78#define DEBUG_TYPE "dagcombine"
80STATISTIC(NodesCombined ,
"Number of dag nodes combined");
81STATISTIC(PreIndexedNodes ,
"Number of pre-indexed nodes created");
82STATISTIC(PostIndexedNodes,
"Number of post-indexed nodes created");
83STATISTIC(OpsNarrowed ,
"Number of load/op/store narrowed");
84STATISTIC(LdStFP2Int ,
"Number of fp load/store pairs transformed to int");
86STATISTIC(NumFPLogicOpsConv,
"Number of logic ops converted to fp ops");
90 cl::desc(
"Enable DAG combiner's use of IR alias analysis"));
94 cl::desc(
"Enable DAG combiner's use of TBAA"));
99 cl::desc(
"Only use DAG-combiner alias analysis in this"
107 cl::desc(
"Bypass the profitability model of load slicing"),
112 cl::desc(
"DAG combiner may split indexing from loads"));
116 cl::desc(
"DAG combiner enable merging multiple stores "
117 "into a wider store"));
121 cl::desc(
"Limit the number of operands to inline for Token Factors"));
125 cl::desc(
"Limit the number of times for the same StoreNode and RootNode "
126 "to bail out in store merging dependence check"));
130 cl::desc(
"DAG combiner enable reducing the width of load/op/store "
135 cl::desc(
"DAG combiner enable load/<replace bytes>/store with "
136 "a narrower store"));
146 bool LegalDAG =
false;
147 bool LegalOperations =
false;
148 bool LegalTypes =
false;
150 bool DisableGenericCombines;
191 void AddUsersToWorklist(
SDNode *
N) {
197 void AddToWorklistWithUsers(
SDNode *
N) {
198 AddUsersToWorklist(
N);
205 void clearAddedDanglingWorklistEntries() {
207 while (!PruningList.
empty()) {
210 recursivelyDeleteUnusedNodes(
N);
214 SDNode *getNextWorklistEntry() {
216 clearAddedDanglingWorklistEntries();
220 while (!
N && !Worklist.
empty()) {
225 bool GoodWorklistEntry = WorklistMap.
erase(
N);
226 (
void)GoodWorklistEntry;
227 assert(GoodWorklistEntry &&
228 "Found a worklist entry without a corresponding map entry!");
238 : DAG(
D), TLI(
D.getTargetLoweringInfo()),
239 STI(
D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL), AA(AA) {
243 MaximumLegalStoreInBits = 0;
249 VT.getSizeInBits().getKnownMinSize() >= MaximumLegalStoreInBits)
250 MaximumLegalStoreInBits = VT.getSizeInBits().getKnownMinSize();
253 void ConsiderForPruning(
SDNode *
N) {
260 void AddToWorklist(
SDNode *
N) {
262 "Deleted Node added to Worklist");
266 if (
N->getOpcode() == ISD::HANDLENODE)
269 ConsiderForPruning(
N);
271 if (WorklistMap.
insert(std::make_pair(
N, Worklist.
size())).second)
276 void removeFromWorklist(
SDNode *
N) {
279 StoreRootCountMap.
erase(
N);
281 auto It = WorklistMap.
find(
N);
282 if (It == WorklistMap.
end())
286 Worklist[It->second] =
nullptr;
287 WorklistMap.
erase(It);
290 void deleteAndRecombine(
SDNode *
N);
291 bool recursivelyDeleteUnusedNodes(
SDNode *
N);
299 return CombineTo(
N, &Res, 1, AddTo);
306 return CombineTo(
N, To, 2, AddTo);
312 unsigned MaximumLegalStoreInBits;
317 bool SimplifyDemandedBits(
SDValue Op) {
318 unsigned BitWidth =
Op.getScalarValueSizeInBits();
330 AddToWorklist(
Op.getNode());
332 CommitTargetLoweringOpt(TLO);
339 bool SimplifyDemandedVectorElts(
SDValue Op) {
341 if (
Op.getValueType().isScalableVector())
344 unsigned NumElts =
Op.getValueType().getVectorNumElements();
346 return SimplifyDemandedVectorElts(Op, DemandedElts);
350 const APInt &DemandedElts,
351 bool AssumeSingleUse =
false);
352 bool SimplifyDemandedVectorElts(
SDValue Op,
const APInt &DemandedElts,
353 bool AssumeSingleUse =
false);
355 bool CombineToPreIndexedLoadStore(
SDNode *
N);
356 bool CombineToPostIndexedLoadStore(
SDNode *
N);
376 void ReplaceLoadWithPromotedLoad(
SDNode *Load,
SDNode *ExtLoad);
522 bool reassociationCanBreakAddressingModePattern(
unsigned Opc,
542 bool NotExtCompare =
false);
543 SDValue convertSelectOfFPConstantsToLoadOffset(
557 const SDLoc &
DL,
bool foldBooleans);
561 SDValue &CC,
bool MatchStrict =
false)
const;
562 bool isOneUseSetCC(
SDValue N)
const;
587 bool DemandHighBits =
true);
591 unsigned PosOpcode,
unsigned NegOpcode,
595 unsigned PosOpcode,
unsigned NegOpcode,
610 SDValue VecIn2,
unsigned LeftIdx,
645 int64_t OffsetFromBase;
648 : MemNode(
N), OffsetFromBase(
Offset) {}
653 StoreSource getStoreSource(
SDValue StoreVal) {
657 return StoreSource::Constant;
660 return StoreSource::Extract;
662 return StoreSource::Load;
664 return StoreSource::Unknown;
672 bool isMulAddWithConstProfitable(
SDNode *MulNode,
SDValue AddNode,
679 EVT LoadResultTy,
EVT &ExtVT);
684 EVT &MemVT,
unsigned ShAmt = 0);
692 bool BackwardsPropagateMask(
SDNode *
N);
704 EVT MemVT,
unsigned NumStores,
705 bool IsConstantSrc,
bool UseVector,
719 bool checkMergeStoreCandidatesForDependencies(
727 int64_t ElementSizeBytes)
const;
732 unsigned NumConsecutiveStores,
733 EVT MemVT,
SDNode *Root,
bool AllowVectors);
740 unsigned NumConsecutiveStores,
EVT MemVT,
746 unsigned NumConsecutiveStores,
EVT MemVT,
747 SDNode *Root,
bool AllowVectors,
748 bool IsNonTemporalStore,
bool IsNonTemporalLoad);
767 bool hasOperation(
unsigned Opcode,
EVT VT) {
779 EVT getShiftAmountTy(
EVT LHSTy) {
786 bool isTypeLegal(
const EVT &VT) {
787 if (!LegalTypes)
return true;
792 EVT getSetCCResultType(
EVT VT)
const {
807 explicit WorklistRemover(DAGCombiner &dc)
808 :
SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
811 DC.removeFromWorklist(
N);
819 explicit WorklistInserter(DAGCombiner &dc)
820 :
SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
824 void NodeInserted(
SDNode *
N)
override { DC.ConsiderForPruning(
N); }
834 ((DAGCombiner*)
DC)->AddToWorklist(
N);
839 return ((DAGCombiner*)
DC)->CombineTo(
N, &To[0], To.
size(), AddTo);
844 return ((DAGCombiner*)
DC)->CombineTo(
N, Res, AddTo);
849 return ((DAGCombiner*)
DC)->CombineTo(
N, Res0, Res1, AddTo);
854 return ((DAGCombiner*)
DC)->recursivelyDeleteUnusedNodes(
N);
859 return ((DAGCombiner*)
DC)->CommitTargetLoweringOpt(TLO);
866void DAGCombiner::deleteAndRecombine(
SDNode *
N) {
867 removeFromWorklist(
N);
875 if (Op->hasOneUse() || Op->getNumValues() > 1)
876 AddToWorklist(Op.getNode());
885 unsigned Bits =
Offset + std::max(LHS.getBitWidth(), RHS.getBitWidth());
886 LHS = LHS.zext(Bits);
887 RHS = RHS.zext(Bits);
896 SDValue &CC,
bool MatchStrict)
const {
898 LHS =
N.getOperand(0);
899 RHS =
N.getOperand(1);
907 LHS =
N.getOperand(1);
908 RHS =
N.getOperand(2);
921 LHS =
N.getOperand(0);
922 RHS =
N.getOperand(1);
930bool DAGCombiner::isOneUseSetCC(
SDValue N)
const {
932 if (isSetCCEquivalent(
N, N0, N1, N2) &&
N->hasOneUse())
947 MaskForTy = 0xFFFFULL;
950 MaskForTy = 0xFFFFFFFFULL;
969 return !(Const->isOpaque() && NoOpaques);
972 unsigned BitWidth =
N.getScalarValueSizeInBits();
973 for (
const SDValue &Op :
N->op_values()) {
977 if (!Const || Const->getAPIntValue().getBitWidth() !=
BitWidth ||
978 (Const->isOpaque() && NoOpaques))
1000bool DAGCombiner::reassociationCanBreakAddressingModePattern(
unsigned Opc,
1021 const APInt &C2APIntVal = C2->getAPIntValue();
1029 const APInt &C1APIntVal = C1->getAPIntValue();
1030 const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal;
1033 const int64_t CombinedValue = CombinedValueIntVal.
getSExtValue();
1042 AM.
BaseOffs = C2APIntVal.getSExtValue();
1044 unsigned AS =
LoadStore->getAddressSpace();
1069 AM.
BaseOffs = C2APIntVal.getSExtValue();
1071 unsigned AS =
LoadStore->getAddressSpace();
1084SDValue DAGCombiner::reassociateOpsCommutative(
unsigned Opc,
const SDLoc &
DL,
1098 return DAG.
getNode(Opc,
DL, VT, N00, OpNode);
1105 return DAG.
getNode(Opc,
DL, VT, OpNode, N01);
1115 if (N1 == N00 || N1 == N01)
1160 if (!Flags.hasAllowReassociation() || !Flags.hasNoSignedZeros())
1163 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N0, N1))
1165 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N1, N0))
1172 assert(
N->getNumValues() == NumTo &&
"Broken CombineTo call!");
1176 dbgs() <<
" and " << NumTo - 1 <<
" other values\n");
1177 for (
unsigned i = 0, e = NumTo; i !=
e; ++i)
1178 assert((!To[i].getNode() ||
1179 N->getValueType(i) == To[i].getValueType()) &&
1180 "Cannot combine value to value of different type!");
1182 WorklistRemover DeadNodes(*
this);
1186 for (
unsigned i = 0, e = NumTo; i !=
e; ++i) {
1187 if (To[i].getNode())
1188 AddToWorklistWithUsers(To[i].getNode());
1196 deleteAndRecombine(
N);
1209 WorklistRemover DeadNodes(*
this);
1225 const APInt &DemandedElts,
1226 bool AssumeSingleUse) {
1234 AddToWorklist(
Op.getNode());
1236 CommitTargetLoweringOpt(TLO);
1243bool DAGCombiner::SimplifyDemandedVectorElts(
SDValue Op,
1244 const APInt &DemandedElts,
1245 bool AssumeSingleUse) {
1247 APInt KnownUndef, KnownZero;
1249 TLO, 0, AssumeSingleUse))
1253 AddToWorklist(
Op.getNode());
1255 CommitTargetLoweringOpt(TLO);
1259void DAGCombiner::ReplaceLoadWithPromotedLoad(
SDNode *Load,
SDNode *ExtLoad) {
1261 EVT VT =
Load->getValueType(0);
1266 WorklistRemover DeadNodes(*
this);
1269 deleteAndRecombine(Load);
1270 AddToWorklist(Trunc.
getNode());
1278 EVT MemVT =
LD->getMemoryVT();
1280 :
LD->getExtensionType();
1283 LD->getChain(),
LD->getBasePtr(),
1284 MemVT,
LD->getMemOperand());
1287 unsigned Opc =
Op.getOpcode();
1291 if (
SDValue Op0 = SExtPromoteOperand(
Op.getOperand(0), PVT))
1295 if (
SDValue Op0 = ZExtPromoteOperand(
Op.getOperand(0), PVT))
1301 return DAG.
getNode(ExtOpc,
DL, PVT, Op);
1313 EVT OldVT =
Op.getValueType();
1315 bool Replace =
false;
1316 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1319 AddToWorklist(NewOp.
getNode());
1322 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1328 EVT OldVT =
Op.getValueType();
1330 bool Replace =
false;
1331 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1334 AddToWorklist(NewOp.
getNode());
1337 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1345 if (!LegalOperations)
1348 EVT VT =
Op.getValueType();
1354 unsigned Opc =
Op.getOpcode();
1362 assert(PVT != VT &&
"Don't know what type to promote to!");
1366 bool Replace0 =
false;
1368 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1370 bool Replace1 =
false;
1372 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1384 Replace1 &= (N0 != N1) && !N1->
hasOneUse();
1387 CombineTo(
Op.getNode(), RV);
1413 if (!LegalOperations)
1416 EVT VT =
Op.getValueType();
1422 unsigned Opc =
Op.getOpcode();
1430 assert(PVT != VT &&
"Don't know what type to promote to!");
1434 bool Replace =
false;
1437 N0 = SExtPromoteOperand(N0, PVT);
1439 N0 = ZExtPromoteOperand(N0, PVT);
1441 N0 = PromoteOperand(N0, PVT, Replace);
1452 ReplaceLoadWithPromotedLoad(
Op.getOperand(0).getNode(), N0.
getNode());
1462 if (!LegalOperations)
1465 EVT VT =
Op.getValueType();
1471 unsigned Opc =
Op.getOpcode();
1479 assert(PVT != VT &&
"Don't know what type to promote to!");
1489bool DAGCombiner::PromoteLoad(
SDValue Op) {
1490 if (!LegalOperations)
1496 EVT VT =
Op.getValueType();
1502 unsigned Opc =
Op.getOpcode();
1510 assert(PVT != VT &&
"Don't know what type to promote to!");
1515 EVT MemVT =
LD->getMemoryVT();
1517 :
LD->getExtensionType();
1519 LD->getChain(),
LD->getBasePtr(),
1520 MemVT,
LD->getMemOperand());
1525 WorklistRemover DeadNodes(*
this);
1528 deleteAndRecombine(
N);
1529 AddToWorklist(
Result.getNode());
1541bool DAGCombiner::recursivelyDeleteUnusedNodes(
SDNode *
N) {
1542 if (!
N->use_empty())
1552 if (
N->use_empty()) {
1554 Nodes.
insert(ChildN.getNode());
1556 removeFromWorklist(
N);
1561 }
while (!Nodes.
empty());
1576 WorklistInserter AddNodes(*
this);
1580 AddToWorklist(&
Node);
1588 while (
SDNode *
N = getNextWorklistEntry()) {
1592 if (recursivelyDeleteUnusedNodes(
N))
1595 WorklistRemover DeadNodes(*
this);
1603 for (
SDNode *LN : UpdatedNodes)
1604 AddToWorklistWithUsers(LN);
1617 if (!CombinedNodes.
count(ChildN.getNode()))
1618 AddToWorklist(ChildN.getNode());
1636 "Node was deleted but visit returned new node!");
1644 N->getNumValues() == 1 &&
"Type mismatch");
1655 AddUsersToWorklist(RV.
getNode());
1662 recursivelyDeleteUnusedNodes(
N);
1671 switch (
N->getOpcode()) {
1751 case ISD::BITCAST:
return visitBITCAST(
N);
1760 case ISD::FSQRT:
return visitFSQRT(
N);
1762 case ISD::FPOW:
return visitFPOW(
N);
1768 case ISD::FP_EXTEND:
return visitFP_EXTEND(
N);
1769 case ISD::FNEG:
return visitFNEG(
N);
1770 case ISD::FABS:
return visitFABS(
N);
1771 case ISD::FFLOOR:
return visitFFLOOR(
N);
1775 case ISD::FMAXIMUM:
return visitFMinMax(
N);
1776 case ISD::FCEIL:
return visitFCEIL(
N);
1777 case ISD::FTRUNC:
return visitFTRUNC(
N);
1778 case ISD::BRCOND:
return visitBRCOND(
N);
1779 case ISD::BR_CC:
return visitBR_CC(
N);
1780 case ISD::LOAD:
return visitLOAD(
N);
1781 case ISD::STORE:
return visitSTORE(
N);
1790 case ISD::MGATHER:
return visitMGATHER(
N);
1791 case ISD::MLOAD:
return visitMLOAD(
N);
1792 case ISD::MSCATTER:
return visitMSCATTER(
N);
1793 case ISD::MSTORE:
return visitMSTORE(
N);
1794 case ISD::LIFETIME_END:
return visitLIFETIME_END(
N);
1795 case ISD::FP_TO_FP16:
return visitFP_TO_FP16(
N);
1796 case ISD::FP16_TO_FP:
return visitFP16_TO_FP(
N);
1797 case ISD::FP_TO_BF16:
return visitFP_TO_BF16(
N);
1799 case ISD::VECREDUCE_FADD:
1800 case ISD::VECREDUCE_FMUL:
1801 case ISD::VECREDUCE_ADD:
1802 case ISD::VECREDUCE_MUL:
1803 case ISD::VECREDUCE_AND:
1804 case ISD::VECREDUCE_OR:
1805 case ISD::VECREDUCE_XOR:
1806 case ISD::VECREDUCE_SMAX:
1807 case ISD::VECREDUCE_SMIN:
1808 case ISD::VECREDUCE_UMAX:
1809 case ISD::VECREDUCE_UMIN:
1810 case ISD::VECREDUCE_FMAX:
1811 case ISD::VECREDUCE_FMIN:
return visitVECREDUCE(
N);
1812#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) case ISD::SDOPC:
1813#include "llvm/IR/VPIntrinsics.def"
1814 return visitVPOp(
N);
1821 if (!DisableGenericCombines)
1827 "Node was deleted but visit returned NULL!");
1834 DagCombineInfo(DAG, Level,
false,
this);
1842 switch (
N->getOpcode()) {
1850 RV = PromoteIntBinOp(
SDValue(
N, 0));
1855 RV = PromoteIntShiftOp(
SDValue(
N, 0));
1891 if (
unsigned NumOps =
N->getNumOperands()) {
1892 if (
N->getOperand(0).getValueType() ==
MVT::Other)
1893 return N->getOperand(0);
1894 if (
N->getOperand(NumOps-1).getValueType() ==
MVT::Other)
1895 return N->getOperand(NumOps-1);
1896 for (
unsigned i = 1; i < NumOps-1; ++i)
1897 if (
N->getOperand(i).getValueType() ==
MVT::Other)
1898 return N->getOperand(i);
1906 if (
N->getNumOperands() == 2) {
1908 return N->getOperand(0);
1910 return N->getOperand(1);
1925 AddToWorklist(*(
N->use_begin()));
1937 for (
unsigned i = 0; i < TFs.
size(); ++i) {
1942 for (
unsigned j = i; j < TFs.
size(); j++)
1953 switch (
Op.getOpcode()) {
1971 if (SeenOps.
insert(
Op.getNode()).second)
1982 for (
unsigned i = 1, e = TFs.
size(); i < e; i++)
1983 AddToWorklist(TFs[i]);
1995 bool DidPruneOps =
false;
1997 unsigned NumLeftToConsider = 0;
1998 for (
const SDValue &Op : Ops) {
1999 Worklist.
push_back(std::make_pair(
Op.getNode(), NumLeftToConsider++));
2003 auto AddToWorklist = [&](
unsigned CurIdx,
SDNode *
Op,
unsigned OpNumber) {
2009 unsigned OrigOpNumber = 0;
2010 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() != Op)
2012 assert((OrigOpNumber != Ops.size()) &&
2013 "expected to find TokenFactor Operand");
2015 for (
unsigned i = CurIdx + 1; i < Worklist.
size(); ++i) {
2016 if (Worklist[i].second == OrigOpNumber) {
2017 Worklist[i].second = OpNumber;
2020 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
2021 OpWorkCount[OrigOpNumber] = 0;
2022 NumLeftToConsider--;
2025 if (SeenChains.
insert(Op).second) {
2026 OpWorkCount[OpNumber]++;
2027 Worklist.
push_back(std::make_pair(Op, OpNumber));
2031 for (
unsigned i = 0; i < Worklist.
size() && i < 1024; ++i) {
2033 if (NumLeftToConsider <= 1)
2035 auto CurNode = Worklist[i].first;
2036 auto CurOpNumber = Worklist[i].second;
2037 assert((OpWorkCount[CurOpNumber] > 0) &&
2038 "Node should not appear in worklist");
2039 switch (CurNode->getOpcode()) {
2045 NumLeftToConsider++;
2049 AddToWorklist(i,
Op.getNode(), CurOpNumber);
2051 case ISD::LIFETIME_START:
2052 case ISD::LIFETIME_END:
2055 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
2059 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
2062 OpWorkCount[CurOpNumber]--;
2063 if (OpWorkCount[CurOpNumber] == 0)
2064 NumLeftToConsider--;
2077 for (
const SDValue &Op : Ops) {
2078 if (SeenChains.
count(
Op.getNode()) == 0)
2093 WorklistRemover DeadNodes(*
this);
2099 AddUsersToWorklist(
N);
2103 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i)
2106 }
while (!
N->use_empty());
2107 deleteAndRecombine(
N);
2115 return Const !=
nullptr && !Const->isOpaque() ? Const :
nullptr;
2126 if (LD->isIndexed() || LD->getBasePtr().getNode() !=
N)
2128 VT = LD->getMemoryVT();
2129 AS = LD->getAddressSpace();
2131 if (ST->isIndexed() || ST->getBasePtr().getNode() !=
N)
2133 VT = ST->getMemoryVT();
2134 AS = ST->getAddressSpace();
2136 if (LD->isIndexed() || LD->getBasePtr().getNode() !=
N)
2138 VT = LD->getMemoryVT();
2139 AS = LD->getAddressSpace();
2141 if (ST->isIndexed() || ST->getBasePtr().getNode() !=
N)
2143 VT = ST->getMemoryVT();
2144 AS = ST->getAddressSpace();
2159 }
else if (
N->getOpcode() ==
ISD::SUB) {
2181 bool ShouldCommuteOperands) {
2186 if (ShouldCommuteOperands)
2193 unsigned Opcode =
N->getOpcode();
2194 EVT VT =
N->getValueType(0);
2202 auto isIdentityConstantForOpcode = [](
unsigned Opcode,
SDValue V) {
2206 return C->isZero() &&
C->isNegative();
2208 return C->isZero() && !
C->isNegative();
2211 return C->isExactlyValue(1.0);
2231 if (isIdentityConstantForOpcode(Opcode, TVal)) {
2237 if (isIdentityConstantForOpcode(Opcode, FVal)) {
2248 "Unexpected binary operator");
2265 unsigned SelOpNo = 0;
2290 bool CanFoldNonConst =
2296 if (!CanFoldNonConst &&
2306 : DAG.
getNode(BinOpcode,
DL, VT, CT, CBO);
2307 if (!CanFoldNonConst && !NewCT.
isUndef() &&
2313 : DAG.
getNode(BinOpcode,
DL, VT, CF, CBO);
2314 if (!CanFoldNonConst && !NewCF.
isUndef() &&
2326 "Expecting add or sub");
2331 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2332 SDValue C = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2333 SDValue Z = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2340 Z.getOperand(0).getValueType() !=
MVT::i1)
2355 EVT VT =
C.getValueType();
2367 "Expecting add or sub");
2371 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2372 SDValue ConstantOp = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2373 SDValue ShiftOp = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2396 {ConstantOp, DAG.getConstant(1, DL, VT)})) {
2406 unsigned Opcode = V.getOpcode();
2440 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
2472 if ((!LegalOperations ||
2475 X.getScalarValueSizeInBits() == 1) {
2491 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
2495 if (!reassociationCanBreakAddressingModePattern(
ISD::ADD,
DL,
N, N0, N1)) {
2581 return (!Max && !Op) ||
2582 (
Max &&
Op &&
Max->getAPIntValue() == (-
Op->getAPIntValue()));
2634 if (
SDValue Combined = visitADDLikeCommutative(N0, N1,
N))
2637 if (
SDValue Combined = visitADDLikeCommutative(N1, N0,
N))
2649 if (
SDValue Combined = visitADDLike(
N))
2685 APInt NewStep = C0 + C1;
2695 APInt NewStep = SV0 + SV1;
2704 unsigned Opcode =
N->getOpcode();
2721 return DAG.
getNode(Opcode,
DL, VT, N1, N0);
2725 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
2746 bool Masked =
false;
2751 V = V.getOperand(0);
2757 V = V.getOperand(0);
2765 if (V.getResNo() != 1)
2772 EVT VT = V->getValueType(0);
2878 DAG.
getVTList(VT, Carry.getValueType()), N0,
2891 if (!
N->hasAnyUseOfValue(1))
2934 EVT VT = V.getValueType();
2936 bool IsFlip =
false;
2939 IsFlip = Const->isOne();
2942 IsFlip = Const->isAllOnes();
2945 IsFlip = (Const->getAPIntValue() & 0x01) == 1;
2950 return V.getOperand(0);
2962 EVT CarryVT =
N->getValueType(1);
2966 if (!
N->hasAnyUseOfValue(1))
2973 return DAG.
getNode(
N->getOpcode(),
DL,
N->getVTList(), N1, N0);
2993 if (
SDValue Combined = visitUADDOLike(N0, N1,
N))
2996 if (
SDValue Combined = visitUADDOLike(N1, N0,
N))
3060 if (!LegalOperations ||
3070 AddToWorklist(CarryExt.
getNode());
3076 if (
SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn,
N))
3079 if (
SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn,
N))
3099 if (!LegalOperations ||
3246 unsigned CarryInOperandNum =
3248 if (Opcode ==
ISD::USUBO && CarryInOperandNum != 1)
3333 "Illegal truncation");
3359 !(!LegalOperations || hasOperation(
ISD::USUBSAT, DstVT)))
3362 EVT SubVT =
N->getValueType(0);
3420 auto PeekThroughFreeze = [](
SDValue N) {
3422 return N->getOperand(0);
3428 if (PeekThroughFreeze(N0) == PeekThroughFreeze(N1))
3437 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
3445 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3472 if (
N->getFlags().hasNoUnsignedWrap())
3478 if (
N->getFlags().hasNoSignedWrap())
3624 if (
SDValue V = foldSubToUSubSat(VT,
N))
3686 if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0))
3703 if (GA->getGlobal() == GB->getGlobal())
3759 DAG.
getVTList(VT, Carry.getValueType()), NegX, Zero,
3767 if (!C0->isOpaque()) {
3768 const APInt &C0Val = C0->getAPIntValue();
3769 const APInt &MaybeOnes = ~DAG.computeKnownBits(N1).Zero;
3770 if ((C0Val - MaybeOnes) == (C0Val ^ MaybeOnes))
3798 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
3820 if (!
N->hasAnyUseOfValue(1))
3847 EVT CarryVT =
N->getValueType(1);
3851 if (!
N->hasAnyUseOfValue(1))
3899 if (!LegalOperations ||
3914 if (!LegalOperations ||
3965 bool N1IsConst =
false;
3966 bool N1IsOpaqueConst =
false;
3971 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
3977 "Splat APInt should be element width");
3987 if (N1IsConst && ConstValue1.
isZero())
3991 if (N1IsConst && ConstValue1.
isOne())
3994 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3998 if (N1IsConst && ConstValue1.
isAllOnes())
4006 SDValue LogBase2 = BuildLogBase2(N1,
DL);
4014 unsigned Log2Val = (-ConstValue1).logBase2();
4048 if ((MulC - 1).isPowerOf2())
4050 else if ((MulC + 1).isPowerOf2())
4055 MathOp ==
ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2();
4058 "multiply-by-constant generated out of bounds shift");
4062 TZeros ? DAG.
getNode(MathOp,
DL, VT, Shl,
4114 const APInt &C1 = NC1->getAPIntValue();
4123 APInt NewStep = C0 * MulVal;
4136 if (!V || V->isZero()) {
4150 for (
unsigned I = 0;
I != NumElts; ++
I)
4172 EVT NodeType =
Node->getValueType(0);
4173 if (!NodeType.isSimple())
4175 switch (NodeType.getSimpleVT().SimpleTy) {
4176 default:
return false;
4177 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
4178 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
4179 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
4180 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
4181 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
4189 if (
Node->use_empty())
4197 EVT VT =
Node->getValueType(0);
4211 unsigned OtherOpcode = 0;
4232 unsigned UserOpc =
User->getOpcode();
4233 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
4237 if (UserOpc == OtherOpcode) {
4240 }
else if (UserOpc == DivRemOpc) {
4243 assert(UserOpc == Opcode);
4248 CombineTo(
User, combined);
4259 EVT VT =
N->getValueType(0);
4262 unsigned Opc =
N->getOpcode();
4271 if (DAG.
isUndef(Opc, {N0, N1}))
4282 if (N0C && N0C->
isZero())
4305 EVT VT =
N->getValueType(0);
4315 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4332 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4340 if (
SDValue V = visitSDIVLike(N0, N1,
N)) {
4347 AddToWorklist(
Mul.getNode());
4349 CombineTo(RemNode, Sub);
4369 if (
C->isZero() ||
C->isOpaque())
4371 if (
C->getAPIntValue().isPowerOf2())
4373 if (
C->getAPIntValue().isNegatedPowerOf2())
4383 EVT VT =
N->getValueType(0);
4408 AddToWorklist(Sign.
getNode());
4414 AddToWorklist(
Add.getNode());
4425 Sra = DAG.
getSelect(
DL, VT, IsOneOrAllOnes, N0, Sra);
4453 EVT VT =
N->getValueType(0);
4463 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4476 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4479 if (
SDValue V = visitUDIVLike(N0, N1,
N)) {
4486 AddToWorklist(
Mul.getNode());
4488 CombineTo(RemNode, Sub);
4506 EVT VT =
N->getValueType(0);
4511 SDValue LogBase2 = BuildLogBase2(N1,
DL);
4512 AddToWorklist(LogBase2.
getNode());
4516 AddToWorklist(Trunc.
getNode());
4525 SDValue LogBase2 = BuildLogBase2(N10,
DL);
4526 AddToWorklist(LogBase2.
getNode());
4530 AddToWorklist(Trunc.
getNode());
4532 AddToWorklist(
Add.getNode());
4559 unsigned Opcode =
N->getOpcode();
4562 EVT VT =
N->getValueType(0);
4583 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4596 AddToWorklist(
Add.getNode());
4607 AddToWorklist(
Add.getNode());
4624 if (
SDValue OptimizedRem = buildOptimizedSREM(N0, N1,
N))
4625 return OptimizedRem;
4629 isSigned ? visitSDIVLike(N0, N1,
N) : visitUDIVLike(N0, N1,
N);
4632 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
4633 if (SDNode *DivNode = DAG.getNodeIfExists(DivOpcode, N->getVTList(),
4635 CombineTo(DivNode, OptimizedDiv);
4638 AddToWorklist(OptimizedDiv.
getNode());
4639 AddToWorklist(
Mul.getNode());
4646 return DivRem.getValue(1);
4654 EVT VT =
N->getValueType(0);
4667 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4695 unsigned SimpleSize =
Simple.getSizeInBits();
4714 EVT VT =
N->getValueType(0);
4727 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4752 SDValue LogBase2 = BuildLogBase2(N1,
DL);
4765 unsigned SimpleSize =
Simple.getSizeInBits();
4788 unsigned Opcode =
N->getOpcode();
4791 EVT VT =
N->getValueType(0);
4801 return DAG.
getNode(Opcode,
DL,
N->getVTList(), N1, N0);
4804 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4830SDValue DAGCombiner::SimplifyNodeWithTwoResults(
SDNode *
N,
unsigned LoOp,
4834 if (!HiExists && (!LegalOperations ||
4837 return CombineTo(
N, Res, Res);
4841 bool LoExists =
N->hasAnyUseOfValue(0);
4842 if (!LoExists && (!LegalOperations ||
4845 return CombineTo(
N, Res, Res);
4849 if (LoExists && HiExists)
4855 AddToWorklist(
Lo.getNode());
4858 (!LegalOperations ||
4860 return CombineTo(
N, LoOpt, LoOpt);
4865 AddToWorklist(
Hi.getNode());
4868 (!LegalOperations ||
4870 return CombineTo(
N, HiOpt, HiOpt);
4882 EVT VT =
N->getValueType(0);
4894 unsigned SimpleSize =
Simple.getSizeInBits();
4907 return CombineTo(
N,
Lo,
Hi);
4920 EVT VT =
N->getValueType(0);
4931 return CombineTo(
N, Zero, Zero);
4937 return CombineTo(
N, N0, Zero);
4944 unsigned SimpleSize =
Simple.getSizeInBits();
4957 return CombineTo(
N,
Lo,
Hi);
4970 EVT CarryVT =
N->getValueType(1);
4991 return DAG.
getNode(
N->getOpcode(),
DL,
N->getVTList(), N1, N0);
5003 N->getVTList(), N0, N0);
5009 return CombineTo(
N,
And,
5064 unsigned Opcode0 = isSignedMinMax(N0, N1, N2, N3, CC);
5098 unsigned Opcode1 = isSignedMinMax(N00, N01, N02, N03, N0CC);
5099 if (!Opcode1 || Opcode0 == Opcode1)
5109 APInt MinCPlus1 = MinC + 1;
5110 if (-MaxC == MinCPlus1 && MinCPlus1.
isPowerOf2()) {
5168 unsigned BW = (C1 + 1).exactLogBase2();
5188 unsigned Opcode =
N->getOpcode();
5202 return DAG.
getNode(Opcode,
DL, VT, N1, N0);
5206 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
5223 return DAG.
getNode(AltOpcode,
DL, VT, N0, N1);
5246 unsigned LogicOpcode =
N->getOpcode();
5249 LogicOpcode ==
ISD::XOR) &&
"Expected logic opcode");
5262 EVT XVT =
X.getValueType();
5271 if (XVT !=
Y.getValueType())
5275 if ((VT.
isVector() || LegalOperations) &&
5285 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5295 if (XVT !=
Y.getValueType())
5307 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5328 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5341 if (XVT.
isInteger() && XVT ==
Y.getValueType() &&
5345 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5364 assert(
X.getValueType() ==
Y.getValueType() &&
5365 "Inputs to shuffles are not the same type");
5371 if (!SVN0->hasOneUse() || !SVN1->hasOneUse() ||
5372 !SVN0->getMask().equals(SVN1->getMask()))
5408 SDValue LL, LR, RL, RR, N0CC, N1CC;
5409 if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
5410 !isSetCCEquivalent(N1, RL, RR, N1CC))
5414 "Unexpected operand types for bitwise logic op");
5417 "Unexpected operand types for setcc");
5433 if (LR == RR && CC0 == CC1 && IsInteger) {
5438 bool AndEqZero = IsAnd && CC1 ==
ISD::SETEQ && IsZero;
5440 bool AndGtNeg1 = IsAnd && CC1 ==
ISD::SETGT && IsNeg1;
5442 bool OrNeZero = !IsAnd && CC1 ==
ISD::SETNE && IsZero;
5444 bool OrLtZero = !IsAnd && CC1 ==
ISD::SETLT && IsZero;
5450 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
5452 AddToWorklist(
Or.getNode());
5457 bool AndEqNeg1 = IsAnd && CC1 ==
ISD::SETEQ && IsNeg1;
5459 bool AndLtZero = IsAnd && CC1 ==
ISD::SETLT && IsZero;
5461 bool OrNeNeg1 = !IsAnd && CC1 ==
ISD::SETNE && IsNeg1;
5463 bool OrGtNeg1 = !IsAnd && CC1 ==
ISD::SETGT && IsNeg1;
5469 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
5471 AddToWorklist(
And.getNode());
5485 AddToWorklist(
Add.getNode());
5512 return !C0->
isOpaque() && !C1->isOpaque() && (CMax - CMin).isPowerOf2();
5530 if (LL == RR && LR == RL) {
5537 if (LL == RL && LR == RR) {
5541 (!LegalOperations ||
5562 if (
SDValue V = foldLogicOfSetCCs(
true, N0, N1,
DL))
5575 APInt SRLC = SRLI->getAPIntValue();
5576 if (
ADDC.getMinSignedBits() <= 64 &&
5588 CombineTo(N0.
getNode(), NewAdd);
5605 const APInt &AndMask = CAnd->getAPIntValue();
5618 (ShiftBits + MaskBits <=
Size / 2) &&
5651 EVT LoadResultTy,
EVT &ExtVT) {
5660 if (ExtVT == LoadedVT &&
5661 (!LegalOperations ||
5677 if (LegalOperations &&
5713 if (LdStMemVT.
bitsLT(MemVT))
5718 assert(ShAmt % 8 == 0 &&
"ShAmt is byte offset");
5719 const unsigned ByteShAmt = ShAmt / 8;
5740 if (LegalOperations &&
5749 if (
Load->getNumValues() > 2)
5768 if (LegalOperations &&
5775bool DAGCombiner::SearchForAndLoads(
SDNode *
N,
5783 if (
Op.getValueType().isVector())
5789 (
Mask->getAPIntValue() &
C->getAPIntValue()) !=
C->getAPIntValue())
5794 if (!
Op.hasOneUse())
5797 switch(
Op.getOpcode()) {
5801 if (isAndLoadExtLoad(Mask, Load,
Load->getValueType(0), ExtVT) &&
5819 unsigned ActiveBits =
Mask->getAPIntValue().countTrailingOnes();
5823 Op.getOperand(0).getValueType();
5834 if (!SearchForAndLoads(
Op.getNode(), Loads, NodesWithConsts, Mask,
5845 NodeToMask =
Op.getNode();
5848 for (
unsigned i = 0, e = NodeToMask->
getNumValues(); i < e; ++i) {
5849 MVT VT =
SDValue(NodeToMask, i).getSimpleValueType();
5852 NodeToMask =
nullptr;
5864bool DAGCombiner::BackwardsPropagateMask(
SDNode *
N) {
5869 if (!
Mask->getAPIntValue().isMask())
5878 SDNode *FixupNode =
nullptr;
5879 if (SearchForAndLoads(
N, Loads, NodesWithConsts, Mask, FixupNode)) {
5880 if (Loads.
size() == 0)
5892 SDValue(FixupNode, 0), MaskOp);
5894 if (
And.getOpcode() == ISD ::AND)
5899 for (
auto *LogicN : NodesWithConsts) {
5913 for (
auto *Load : Loads) {
5918 if (
And.getOpcode() == ISD ::AND)
5921 SDValue NewLoad = reduceLoadWidth(
And.getNode());
5923 "Shouldn't be masking the load if it can't be narrowed");
5924 CombineTo(Load, NewLoad, NewLoad.
getValue(1));
5937SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(
SDNode *
N) {
5948 unsigned OuterShift;
5949 unsigned InnerShift;
5951 auto matchMask = [&OuterShift, &InnerShift, &
Y](
SDValue M) ->
bool {
5954 OuterShift = M->getOpcode();
5963 Y = M->getOperand(1);
5970 else if (matchMask(N0))
5976 EVT VT =
N->getValueType(0);
5993 EVT VT =
And->getValueType(0);
6009 bool FoundNot =
false;
6012 Src = Src.getOperand(0);
6018 Src = Src.getOperand(0);
6022 if (Src.getOpcode() !=
ISD::SRL || !Src.hasOneUse())
6029 SDValue ShiftAmt = Src.getOperand(1);
6031 if (!ShiftAmtC || !ShiftAmtC->getAPIntValue().ult(VTBitWidth))
6035 Src = Src.getOperand(0);
6042 Src = Src.getOperand(0);
6101 unsigned LogicOpcode =
N->getOpcode();
6104 &&
"Expected bitwise logic operation");
6106 if (!LogicOp.hasOneUse() || !ShiftOp.
hasOneUse())
6110 unsigned ShiftOpcode = ShiftOp.
getOpcode();
6111 if (LogicOp.getOpcode() != LogicOpcode ||
6123 if (LogicOp.getOperand(0).getOpcode() == ShiftOpcode &&
6124 LogicOp.getOperand(0).getOperand(1) ==
Y) {
6127 }
else if (LogicOp.getOperand(1).getOpcode() == ShiftOpcode &&
6128 LogicOp.getOperand(1).getOperand(1) ==
Y) {
6135 EVT VT =
N->getValueType(0);
6139 return DAG.
getNode(LogicOpcode,
DL, VT, NewShift,
Z);
6180 EVT LoadVT = MLoad->getMemoryVT();
6190 ExtVT,
SDLoc(
N), MLoad->getChain(), MLoad->getBasePtr(),
6191 MLoad->getOffset(), MLoad->getMask(), MLoad->getPassThru(),
6192 LoadVT, MLoad->getMemOperand(), MLoad->getAddressingMode(),
6209 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
6218 if (
SDValue Shuffle = XformToShuffleWithZero(
N))
6226 return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue());
6246 CombineTo(N0.
getNode(), Zext);
6273 APInt SplatValue, SplatUndef;
6274 unsigned SplatBitSize;
6276 bool IsSplat =
Vector->isConstantSplat(SplatValue, SplatUndef,
6277 SplatBitSize, HasAnyUndefs);
6281 SplatValue |= SplatUndef;
6287 unsigned EltBitWidth =
Vector->getValueType(0).getScalarSizeInBits();
6292 if (EltBitWidth > SplatBitSize)
6293 for (SplatValue = SplatValue.
zextOrTrunc(EltBitWidth);
6294 SplatBitSize < EltBitWidth; SplatBitSize = SplatBitSize * 2)
6295 SplatValue |= SplatValue.
shl(SplatBitSize);
6299 if ((SplatBitSize % EltBitWidth) == 0) {
6301 for (
unsigned i = 0, n = (SplatBitSize / EltBitWidth); i < n; ++i)
6311 Load->getValueType(0),
6312 Load->getMemoryVT());
6320 switch (
Load->getExtensionType()) {
6321 default:
B =
false;
break;
6333 CombineTo(
N, (N0.
getNode() == Load) ? NewLoad : N0);
6338 Load->getChain(),
Load->getBasePtr(),
6339 Load->getOffset(),
Load->getMemoryVT(),
6340 Load->getMemOperand());
6342 if (
Load->getNumValues() == 3) {
6344 SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
6345 NewLoad.getValue(2) };
6346 CombineTo(Load, To, 3,
true);
6348 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
6359 EVT ExtVT =
Ext->getValueType(0);
6377 EVT MemVT = GN0->getMemoryVT();
6383 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
6384 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
6390 CombineTo(
N, ZExtLoad);
6391 AddToWorklist(ZExtLoad.
getNode());
6400 if (
SDValue Res = reduceLoadWidth(
N))
6408 if (BackwardsPropagateMask(
N))
6412 if (
SDValue Combined = visitANDLike(N0, N1,
N))
6417 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
6462 ((!LegalOperations && LN0->
isSimple()) ||
6480 if (
SDValue Shifts = unfoldExtremeBitClearingToShifts(
N))
6500 if (!
C->getAPIntValue().isMask(
6501 LHS.getOperand(0).getValueType().getFixedSizeInBits()))
6508 if (IsAndZeroExtMask(N0, N1))
6520 bool DemandHighBits) {
6521 if (!LegalOperations)
6524 EVT VT =
N->getValueType(0);
6531 bool LookPassAnd0 =
false;
6532 bool LookPassAnd1 =
false;
6547 LookPassAnd0 =
true;
6557 LookPassAnd1 =
true;
6583 LookPassAnd0 =
true;
6597 LookPassAnd1 =
true;
6606 if (OpSizeInBits > 16) {
6610 if (DemandHighBits && !LookPassAnd0)
6617 if (!LookPassAnd1) {
6618 unsigned HighBit = DemandHighBits ? OpSizeInBits : 24;
6626 if (OpSizeInBits > 16) {
6642 if (!
N->hasOneUse())
6645 unsigned Opc =
N.getOpcode();
6663 unsigned MaskByteOffset;
6667 case 0xFF: MaskByteOffset = 0;
break;
6668 case 0xFF00: MaskByteOffset = 1;
break;
6677 case 0xFF0000: MaskByteOffset = 2;
break;
6678 case 0xFF000000: MaskByteOffset = 3;
break;
6683 if (MaskByteOffset == 0 || MaskByteOffset == 2) {
6689 if (!
C ||
C->getZExtValue() != 8)
6697 if (!
C ||
C->getZExtValue() != 8)
6703 if (MaskByteOffset != 0 && MaskByteOffset != 2)
6706 if (!
C ||
C->getZExtValue() != 8)
6711 if (MaskByteOffset != 1 && MaskByteOffset != 3)
6714 if (!
C ||
C->getZExtValue() != 8)
6718 if (Parts[MaskByteOffset])
6733 if (!
C ||
C->getAPIntValue() != 16)
6735 Parts[0] = Parts[1] =
N.getOperand(0).getOperand(0).getNode();
6750 "MatchBSwapHWordOrAndAnd: expecting i32");
6760 if (!Mask0 || !Mask1)
6771 if (!ShiftAmt0 || !ShiftAmt1)
6791 if (!LegalOperations)
6794 EVT VT =
N->getValueType(0);
6834 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
6863 if (
SDValue V = foldLogicOfSetCCs(
false, N0, N1,
DL))
6878 const APInt &LHSMask = N0O1C->getAPIntValue();
6879 const APInt &RHSMask = N1O1C->getAPIntValue();
6929 auto peekThroughZext = [](
SDValue V) {
6931 return V->getOperand(0);
6992 if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
6993 assert((!ZeroN00 || !ZeroN01) &&
"Both inputs zero!");
6994 assert((!ZeroN10 || !ZeroN11) &&
"Both inputs zero!");
6995 bool CanFold =
true;
6999 for (
int i = 0; i != NumElts; ++i) {
7000 int M0 = SV0->getMaskElt(i);
7001 int M1 = SV1->getMaskElt(i);
7004 bool M0Zero =
M0 < 0 || (ZeroN00 == (
M0 < NumElts));
7005 bool M1Zero =
M1 < 0 || (ZeroN10 == (
M1 < NumElts));
7009 if ((M0Zero &&
M1 < 0) || (M1Zero &&
M0 < 0))
7013 if (M0Zero == M1Zero) {
7018 assert((
M0 >= 0 ||
M1 >= 0) &&
"Undef index!");
7024 Mask[i] = M1Zero ?
M0 % NumElts : (
M1 % NumElts) + NumElts;
7035 return LegalShuffle;
7049 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
7057 if (
SDValue Combined = visitORLike(N0, N1,
N))
7064 if (
SDValue BSwap = MatchBSwapHWord(
N, N0, N1))
7066 if (
SDValue BSwap = MatchBSwapHWordLow(
N, N0, N1))
7095 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
7102 if (
SDValue Load = MatchLoadCombine(
N))
7112 if (
SDValue Combined = visitADDLike(
N))
7121 Mask = Op.getOperand(1);
7122 return Op.getOperand(0);
7165 assert(OppShift && ExtractFrom &&
"Empty SDValue");
7193 bool IsMulOrDiv =
false;
7196 auto SelectOpcode = [&](
unsigned NeededShift,
unsigned MulOrDivVariant) {
7197 IsMulOrDiv = ExtractFrom.
getOpcode() == MulOrDivVariant;
7198 if (!IsMulOrDiv && ExtractFrom.
getOpcode() != NeededShift)
7200 Opcode = NeededShift;
7250 if (Rem != 0 || ResultAmt != OppLHSAmt)
7256 if (OppLHSAmt != ExtractFromAmt - NeededShiftAmt.
zextOrTrunc(
7265 return DAG.
getNode(Opcode,
DL, ResVT, OppShiftLHS, NewShiftNode);
7319 unsigned MaskLoBits = 0;
7321 unsigned Bits =
Log2_64(EltSize);
7323 if (NegBits >= Bits) {
7346 if (PosBits >= MaskLoBits) {
7368 if ((Pos == NegOp1) ||
7392 return Width.getLoBits(MaskLoBits) == 0;
7393 return Width == EltSize;
7403 SDValue InnerNeg,
bool HasPos,
7404 unsigned PosOpcode,
unsigned NegOpcode,
7416 return DAG.
getNode(HasPos ? PosOpcode : NegOpcode,
DL, VT, Shifted,
7417 HasPos ? Pos : Neg);
7431 SDValue InnerNeg,
bool HasPos,
7432 unsigned PosOpcode,
unsigned NegOpcode,
7444 if (
matchRotateSub(InnerPos, InnerNeg, EltBits, DAG, N0 == N1)) {
7445 return DAG.
getNode(HasPos ? PosOpcode : NegOpcode,
DL, VT, N0, N1,
7446 HasPos ? Pos : Neg);
7453 auto IsBinOpImm = [](
SDValue Op,
unsigned BinOpc,
unsigned Imm) {
7454 if (
Op.getOpcode() != BinOpc)
7463 IsBinOpImm(InnerNeg,
ISD::XOR, EltBits - 1) &&
7472 IsBinOpImm(InnerPos,
ISD::XOR, EltBits - 1) &&
7482 IsBinOpImm(InnerPos,
ISD::XOR, EltBits - 1) &&
7497 EVT VT = LHS.getValueType();
7502 bool HasROTL = hasOperation(
ISD::ROTL, VT);
7503 bool HasROTR = hasOperation(
ISD::ROTR, VT);
7504 bool HasFSHL = hasOperation(
ISD::FSHL, VT);
7505 bool HasFSHR = hasOperation(
ISD::FSHR, VT);
7516 if (LegalOperations && !HasROTL && !HasROTR && !HasFSHL && !HasFSHR)
7521 LHS.getOperand(0).getValueType() == RHS.getOperand(0).getValueType()) {
7522 assert(LHS.getValueType() == RHS.getValueType());
7523 if (
SDValue Rot = MatchRotate(LHS.getOperand(0), RHS.getOperand(0),
DL)) {
7538 if (!LHSShift && !RHSShift)
7553 RHSShift = NewRHSShift;
7558 LHSShift = NewLHSShift;
7561 if (!RHSShift || !LHSShift)
7588 return (LHS->getAPIntValue() + RHS->getAPIntValue()) == EltSizeInBits;
7591 auto ApplyMasks = [&](
SDValue Res) {
7616 if (!IsRotate && !(HasFSHL || HasFSHR)) {
7617 if (TLI.
isTypeLegal(VT) && LHS.hasOneUse() && RHS.hasOneUse() &&
7625 if (CommonOp ==
Or.getOperand(0)) {
7627 Y =
Or.getOperand(1);
7630 if (CommonOp ==
Or.getOperand(1)) {
7632 Y =
Or.getOperand(0);
7639 if (matchOr(LHSShiftArg, RHSShiftArg)) {
7644 }
else if (matchOr(RHSShiftArg, LHSShiftArg)) {
7653 return ApplyMasks(Res);
7666 if (IsRotate && (HasROTL || HasROTR || !(HasFSHL || HasFSHR))) {
7667 bool UseROTL = !LegalOperations || HasROTL;
7669 UseROTL ? LHSShiftAmt : RHSShiftAmt);
7671 bool UseFSHL = !LegalOperations || HasFSHL;
7673 RHSShiftArg, UseFSHL ? LHSShiftAmt : RHSShiftAmt);
7676 return ApplyMasks(Res);
7681 if (!HasROTL && !HasROTR && !HasFSHL && !HasFSHR)
7690 SDValue LExtOp0 = LHSShiftAmt;
7691 SDValue RExtOp0 = RHSShiftAmt;
7704 if (IsRotate && (HasROTL || HasROTR)) {
7706 MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt, LExtOp0,
7712 MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt, RExtOp0,
7719 MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, LHSShiftAmt, RHSShiftAmt,
7725 MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
7737struct ByteProvider {
7742 unsigned ByteOffset = 0;
7744 ByteProvider() =
default;
7747 return ByteProvider(Load, ByteOffset);
7750 static ByteProvider getConstantZero() {
return ByteProvider(
nullptr, 0); }
7752 bool isConstantZero()
const {
return !
Load; }
7753 bool isMemory()
const {
return Load; }
7760 ByteProvider(
LoadSDNode *Load,
unsigned ByteOffset)
7761 :
Load(
Load), ByteOffset(ByteOffset) {}
7779 bool Root =
false) {
7784 if (!Root && !Op.hasOneUse())
7787 assert(Op.getValueType().isScalarInteger() &&
"can't handle other types");
7788 unsigned BitWidth = Op.getValueSizeInBits();
7792 assert(
Index < ByteWidth &&
"invalid index requested");
7795 switch (Op.getOpcode()) {
7804 if (LHS->isConstantZero())
7806 if (RHS->isConstantZero())
7815 uint64_t BitShift = ShiftOp->getZExtValue();
7816 if (BitShift % 8 != 0)
7820 return Index < ByteShift
7821 ? ByteProvider::getConstantZero()
7828 SDValue NarrowOp = Op->getOperand(0);
7830 if (NarrowBitWidth % 8 != 0)
7832 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
7834 if (
Index >= NarrowByteWidth)
7845 if (!L->isSimple() || L->isIndexed())
7848 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
7849 if (NarrowBitWidth % 8 != 0)
7851 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
7853 if (
Index >= NarrowByteWidth)
7857 return ByteProvider::getMemory(L,
Index);
7876 int64_t FirstOffset) {
7878 unsigned Width = ByteOffsets.
size();
7882 bool BigEndian =
true, LittleEndian =
true;
7883 for (
unsigned i = 0; i < Width; i++) {
7884 int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
7887 if (!BigEndian && !LittleEndian)
7891 assert((BigEndian != LittleEndian) &&
"It should be either big endian or"
7897 switch (
Value.getOpcode()) {
7940 EVT MemVT =
N->getMemoryVT();
7942 !
N->isSimple() ||
N->isIndexed())
7952 if (
Store->getMemoryVT() != MemVT || !
Store->isSimple() ||
7956 Chain =
Store->getChain();
7959 if (Stores.
size() < 2)
7964 unsigned NumStores = Stores.
size();
7965 unsigned NarrowNumBits =
N->getMemoryVT().getScalarSizeInBits();
7966 unsigned WideNumBits = NumStores * NarrowNumBits;
7978 for (
auto *Store : Stores) {
7997 if (ShiftAmtC % NarrowNumBits != 0)
8000 Offset = ShiftAmtC / NarrowNumBits;
8007 SourceValue = WideVal;
8014 SourceValue = WideVal;
8022 int64_t ByteOffsetFromBase = 0;
8025 else if (!
Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
8029 if (ByteOffsetFromBase < FirstOffset) {
8031 FirstOffset = ByteOffsetFromBase;
8037 OffsetMap[
Offset] = ByteOffsetFromBase;
8041 assert(FirstStore &&
"First store must be set");
8048 if (!Allowed || !
Fast)
8053 auto checkOffsets = [&](
bool MatchLittleEndian) {
8054 if (MatchLittleEndian) {
8055 for (
unsigned i = 0; i != NumStores; ++i)
8056 if (OffsetMap[i] != i * (NarrowNumBits / 8) + FirstOffset)
8059 for (
unsigned i = 0, j = NumStores - 1; i != NumStores; ++i, --j)
8060 if (OffsetMap[j] != i * (NarrowNumBits / 8) + FirstOffset)
8067 bool NeedBswap =
false;
8068 bool NeedRotate =
false;
8071 if (NarrowNumBits == 8 && checkOffsets(Layout.
isBigEndian()))
8073 else if (NumStores == 2 && checkOffsets(Layout.
isBigEndian()))
8082 "Unexpected store value to merge");
8091 }
else if (NeedRotate) {
8092 assert(WideNumBits % 2 == 0 &&
"Unexpected type for rotate");
8138 "Can only match load combining against OR nodes");
8141 EVT VT =
N->getValueType(0);
8147 auto MemoryByteOffset = [&] (ByteProvider
P) {
8148 assert(
P.isMemory() &&
"Must be a memory byte provider");
8149 unsigned LoadBitWidth =
P.Load->getMemoryVT().getSizeInBits();
8150 assert(LoadBitWidth % 8 == 0 &&
8151 "can only analyze providers for individual bytes not bit");
8152 unsigned LoadByteWidth = LoadBitWidth / 8;
8153 return IsBigEndianTarget
8168 unsigned ZeroExtendedBytes = 0;
8169 for (
int i = ByteWidth - 1; i >= 0; --i) {
8174 if (
P->isConstantZero()) {
8177 if (++ZeroExtendedBytes != (ByteWidth -
static_cast<unsigned>(i)))
8181 assert(
P->isMemory() &&
"provenance should either be memory or zero");
8186 "Must be enforced by calculateByteProvider");
8193 else if (Chain != LChain)
8198 int64_t ByteOffsetFromBase = 0;
8201 else if (!
Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
8205 ByteOffsetFromBase += MemoryByteOffset(*
P);
8206 ByteOffsets[i] = ByteOffsetFromBase;
8209 if (ByteOffsetFromBase < FirstOffset) {
8210 FirstByteProvider =
P;
8211 FirstOffset = ByteOffsetFromBase;
8216 assert(!Loads.
empty() &&
"All the bytes of the value must be loaded from "
8217 "memory, so there must be at least one load which produces the value");
8218 assert(
Base &&
"Base address of the accessed memory location must be set");
8221 bool NeedsZext = ZeroExtendedBytes > 0;
8232 if (LegalOperations &&
8240 makeArrayRef(ByteOffsets).drop_back(ZeroExtendedBytes), FirstOffset);
8244 assert(FirstByteProvider &&
"must be set");
8248 if (MemoryByteOffset(*FirstByteProvider) != 0)
8250 LoadSDNode *FirstLoad = FirstByteProvider->Load;
8257 bool NeedsBswap = IsBigEndianTarget != *IsBigEndian;
8264 if (NeedsBswap && (LegalOperations || NeedsZext) &&
8270 if (NeedsBswap && NeedsZext && LegalOperations &&
8279 if (!Allowed || !
Fast)
8298 SDLoc(
N), LegalOperations))
8326 EVT VT =
N->getValueType(0);
8354 if (!matchAndXor(N0, 0, N1) && !matchAndXor(N0, 1, N1) &&
8355 !matchAndXor(N1, 0, N0) && !matchAndXor(N1, 1, N0))
8429 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
8441 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
8452 if (
SDValue Combined = visitADDLike(
N))
8459 isSetCCEquivalent(N0, LHS, RHS, CC,
true)) {
8461 LHS.getValueType());
8462 if (!LegalOperations ||
8480 CombineTo(
N, SetCC);
8482 recursivelyDeleteUnusedNodes(N0.
getNode());
8493 isSetCCEquivalent(N0.
getOperand(0), LHS, RHS, CC)){
8498 AddToWorklist(V.getNode());
8506 if (isOneUseSetCC(N01) || isOneUseSetCC(N00)) {
8511 return DAG.
getNode(NewOpcode,
DL, VT, N00, N01);
8523 return DAG.
getNode(NewOpcode,
DL, VT, N00, N01);
8547 AddToWorklist(NotX.
getNode());
8558 if ((A0 == S && A1 == S0) || (A1 == S && A0 == S0))
8595 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
8604 if (
SDValue MM = unfoldMaskedMerge(
N))
8624 if (!LogicOp.hasOneUse())
8627 unsigned LogicOpcode = LogicOp.getOpcode();
8633 unsigned ShiftOpcode = Shift->
getOpcode();
8636 assert(C1Node &&
"Expected a shift with constant operand");
8639 const APInt *&ShiftAmtVal) {
8640 if (V.getOpcode() != ShiftOpcode || !V.hasOneUse())
8648 ShiftOp = V.getOperand(0);
8653 if (ShiftAmtVal->getBitWidth() != C1Val.
getBitWidth())
8657 if ((*ShiftAmtVal + C1Val).uge(V.getScalarValueSizeInBits()))
8666 if (matchFirstShift(LogicOp.getOperand(0),
X, C0Val))
8667 Y = LogicOp.getOperand(1);
8668 else if (matchFirstShift(LogicOp.getOperand(1),
X, C0Val))
8669 Y = LogicOp.getOperand(0);
8680 return DAG.
getNode(LogicOpcode,
DL, VT, NewShift1, NewShift2);
8713 switch (LHS.getOpcode()) {
8742 if (!IsShiftByConstant && !IsCopyOrSelect)
8745 if (IsCopyOrSelect &&
N->hasOneUse())
8750 EVT VT =
N->getValueType(0);
8757 return DAG.
getNode(LHS.getOpcode(),
DL, VT, NewShift, NewRHS);
8765 EVT TruncVT =
N->getValueType(0);
8766 if (
N->hasOneUse() &&
N->getOperand(0).hasOneUse() &&
8774 AddToWorklist(Trunc00.
getNode());
8775 AddToWorklist(Trunc01.
getNode());
8787 EVT VT =
N->getValueType(0);
8802 bool OutOfRange =
false;
8804 OutOfRange |=
C->getAPIntValue().uge(Bitsize);
8812 return DAG.
getNode(
N->getOpcode(), dl, VT, N0, Amt);
8817 if (RotAmtC && RotAmtC->getAPIntValue() == 8 &&
8829 return DAG.
getNode(
N->getOpcode(), dl, VT, N0, NewOp1);
8841 bool SameSide = (
N->getOpcode() == NextOp);
8850 CombineOp, dl, ShiftVT, {Norm1, Norm2})) {
8852 ISD::UREM, dl, ShiftVT, {CombinedShift, BitsizeC});
8900 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
8921 APInt c1 = LHS->getAPIntValue();
8922 APInt c2 = RHS->getAPIntValue();
8924 return (c1 + c2).uge(OpSizeInBits);
8931 APInt c1 = LHS->getAPIntValue();
8932 APInt c2 = RHS->getAPIntValue();
8934 return (c1 + c2).ult(OpSizeInBits);
8959 APInt c1 = LHS->getAPIntValue();
8960 APInt c2 = RHS->getAPIntValue();
8962 return c2.
uge(OpSizeInBits - InnerBitwidth) &&
8963 (c1 + c2).uge(OpSizeInBits);
8972 APInt c1 = LHS->getAPIntValue();
8973 APInt c2 = RHS->getAPIntValue();
8975 return c2.
uge(OpSizeInBits - InnerBitwidth) &&
8976 (c1 + c2).ult(OpSizeInBits);
8998 APInt c1 = LHS->getAPIntValue();
8999 APInt c2 = RHS->getAPIntValue();
9010 AddToWorklist(NewSHL.
getNode());
9018 const APInt &LHSC = LHS->getAPIntValue();
9019 const APInt &RHSC = RHS->getAPIntValue();
9020 return LHSC.
ult(OpSizeInBits) && RHSC.
ult(OpSizeInBits) &&
9096 AddToWorklist(Shl0.
getNode());
9097 AddToWorklist(Shl1.
getNode());
9111 if (
SDValue NewSHL = visitShiftByConstant(
N))
9118 const APInt &C1 = NC1->getAPIntValue();
9128 APInt NewStep = C0 << ShlVal;
9143 "SRL or SRA node is required here!");
9165 if (!IsSignExt && !IsZeroExt)
9173 unsigned ActiveBits = IsSignExt
9174 ?
Constant->getAPIntValue().getMinSignedBits()
9175 :
Constant->getAPIntValue().getActiveBits();
9176 if (ActiveBits > NarrowVTSize)
9193 "Cannot have a multiply node with two different operand types.");
9204 if (ShiftAmt != NarrowVTSize)
9245 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
9257 if (!LegalOperations ||
9277 APInt c1 = LHS->getAPIntValue();
9278 APInt c2 = RHS->getAPIntValue();
9280 APInt Sum = c1 + c2;
9282 Sum.
uge(OpSizeInBits) ? (OpSizeInBits - 1) : Sum.getZExtValue();
9292 "Expected matchBinaryPredicate to return one element for "
9296 ShiftValue = ShiftValues[0];
9324 if ((ShiftAmt > 0) &&
9336 N->getValueType(0), Trunc);
9372 DAG.
getConstant(AddC->getAPIntValue().lshr(ShiftAmt).trunc(
9406 if (LargeShift->getAPIntValue() == TruncBits) {
9428 if (
SDValue NewSRA = visitShiftByConstant(
N))
9437 if (
SDValue NarrowLoad = reduceLoadWidth(
N))
9462 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
9475 APInt c1 = LHS->getAPIntValue();
9476 APInt c2 = RHS->getAPIntValue();
9478 return (c1 + c2).uge(OpSizeInBits);
9485 APInt c1 = LHS->getAPIntValue();
9486 APInt c2 = RHS->getAPIntValue();
9488 return (c1 + c2).ult(OpSizeInBits);
9509 if (c1 + OpSizeInBits == InnerShiftSize) {
9511 if (c1 + c2 >= InnerShiftSize)
9521 c1 + c2 < InnerShiftSize) {
9542 const APInt &LHSC = LHS->getAPIntValue();
9543 const APInt &RHSC = RHS->getAPIntValue();
9544 return LHSC.
ult(OpSizeInBits) && RHSC.
ult(OpSizeInBits) &&
9588 AddToWorklist(SmallShift.
getNode());
9615 APInt UnknownBits = ~Known.Zero;
9632 AddToWorklist(
Op.getNode());
9654 if (
SDValue NewSRL = visitShiftByConstant(
N))
9658 if (
SDValue NarrowLoad = reduceLoadWidth(
N))
9678 if (
N->hasOneUse()) {
9680 if (
Use->getOpcode() == ISD::BRCOND)
9685 if (
Use->getOpcode() == ISD::BRCOND)
9699 EVT VT =
N->getValueType(0);
9711 return IsFSHL ? N0 : N1;
9713 auto IsUndefOrZero = [](
SDValue V) {
9722 if (Cst->getAPIntValue().uge(
BitWidth)) {
9728 unsigned ShAmt = Cst->getZExtValue();
9730 return IsFSHL ? N0 : N1;
9736 if (IsUndefOrZero(N0))
9740 if (IsUndefOrZero(N1))
9754 if (LHS && RHS && LHS->isSimple() && RHS->isSimple() &&
9755 LHS->getAddressSpace() == RHS->getAddressSpace() &&
9765 RHS->getAddressSpace(), NewAlign,
9766 RHS->getMemOperand()->getFlags(), &
Fast) &&
9770 AddToWorklist(NewPtr.
getNode());
9772 VT,
DL, RHS->getChain(), NewPtr,
9773 RHS->getPointerInfo().getWithOffset(PtrOff), NewAlign,
9774 RHS->getMemOperand()->getFlags(), RHS->getAAInfo());
9776 WorklistRemover DeadNodes(*
this);
9802 if (N0 == N1 && hasOperation(RotOpc, VT))
9863 EVT VT =
N->getValueType(0);
9888 EVT VT =
N->getValueType(0);
9908 EVT VT =
N->getValueType(0);
9933 if (ShAmt && ShAmt->getAPIntValue().ult(BW) &&
9934 ShAmt->getZExtValue() >= (BW / 2) &&
9935 (ShAmt->getZExtValue() % 16) == 0 && TLI.
isTypeLegal(HalfVT) &&
9937 (!LegalOperations || hasOperation(
ISD::BSWAP, HalfVT))) {
9939 if (
uint64_t NewShAmt = (ShAmt->getZExtValue() - (BW / 2)))
9955 if (ShAmt && ShAmt->getAPIntValue().ult(BW) &&
9956 ShAmt->getZExtValue() % 8 == 0) {
9968 EVT VT =
N->getValueType(0);
9981 EVT VT =
N->getValueType(0);
9998 EVT VT =
N->getValueType(0);
10008 EVT VT =
N->getValueType(0);
10025 EVT VT =
N->getValueType(0);
10035 EVT VT =
N->getValueType(0);
10049 EVT VT = LHS.getValueType();
10061 if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True))
10075 unsigned IEEEOpcode = (LHS == True) ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
10077 return DAG.
getNode(IEEEOpcode,
DL, VT, LHS, RHS);
10079 unsigned Opcode = (LHS == True) ? ISD::FMINNUM : ISD::FMAXNUM;
10081 return DAG.
getNode(Opcode,
DL, VT, LHS, RHS);
10090 unsigned IEEEOpcode = (LHS == True) ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE;
10092 return DAG.
getNode(IEEEOpcode,
DL, VT, LHS, RHS);
10094 unsigned Opcode = (LHS == True) ? ISD::FMAXNUM : ISD::FMINNUM;
10096 return DAG.
getNode(Opcode,
DL, VT, LHS, RHS);
10113 EVT VT =
N->getValueType(0);
10115 VT !=
Cond.getOperand(0).getValueType())
10145 EVT VT =
N->getValueType(0);
10146 EVT CondVT =
Cond.getValueType();
10161 if (CondVT ==
MVT::i1 && !LegalOperations) {
10162 if (C1->
isZero() && C2->isOne()) {
10169 if (C1->
isZero() && C2->isAllOnes()) {
10176 if (C1->
isOne() && C2->isZero()) {
10194 const APInt &C1Val = C1->getAPIntValue();
10195 const APInt &C2Val = C2->getAPIntValue();
10196 if (C1Val - 1 == C2Val) {
10202 if (C1Val + 1 == C2Val) {
10239 C1->
isZero() && C2->isOne()) {
10252 "Expected a (v)select");
10255 EVT VT =
N->getValueType(0);
10288 EVT VT =
N->getValueType(0);
10346 EVT VT =
N->getValueType(0);
10354 if (
SDValue V = foldSelectOfConstants(
N))
10361 if (SimplifySelectOps(
N, N1, N2))
10373 bool normalizeToSequence =
10382 if (normalizeToSequence || !InnerSelect.
use_empty())
10384 InnerSelect, N2, Flags);
10387 recursivelyDeleteUnusedNodes(InnerSelect.
getNode());
10394 Cond1, N1, N2, Flags);
10395 if (normalizeToSequence || !InnerSelect.
use_empty())
10397 InnerSelect, Flags);
10400 recursivelyDeleteUnusedNodes(InnerSelect.
getNode());
10410 if (!normalizeToSequence) {
10416 if (
SDValue Combined = visitANDLike(N0, N1_0,
N)) {
10429 if (!normalizeToSequence) {
10435 if (
SDValue Combined = visitORLike(N0, N2_0,
N))
10473 if (
C && NotC &&
C->getAPIntValue() == ~NotC->getAPIntValue()) {
10493 (!LegalOperations &&
10504 if (
SDValue NewSel = SimplifySelect(
DL, N0, N1, N2))
10509 if (
SDValue BinOp = foldSelectOfBinops(
N))
10522 EVT VT =
N->getValueType(0);
10530 if (LHS->getNumOperands() != 2 || RHS->getNumOperands() != 2)
10539 for (
int i = 0; i < NumElems / 2; ++i) {
10540 if (
Cond->getOperand(i)->isUndef())
10543 if (BottomHalf ==
nullptr)
10545 else if (
Cond->getOperand(i).getNode() != BottomHalf)
10551 for (
int i = NumElems / 2; i < NumElems; ++i) {
10552 if (
Cond->getOperand(i)->isUndef())
10555 if (TopHalf ==
nullptr)
10557 else if (
Cond->getOperand(i).getNode() != TopHalf)
10561 assert(TopHalf && BottomHalf &&
10562 "One half of the selector was all UNDEFs and the other was all the "
10563 "same value. This should have been addressed before this function.");
10566 BottomHalf->
isZero() ? RHS->getOperand(0) : LHS->getOperand(0),
10567 TopHalf->
isZero() ? RHS->getOperand(1) : LHS->getOperand(1));
10582 if (!SplatVal || SplatVal.
getValueType() != BasePtr.getValueType())
10585 BasePtr = SplatVal;
10676 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
10680 Value.getValueType().isInteger() &&
10683 APInt TruncDemandedBits =
10708 Value.getOperand(0).getValueType());
10731 return CombineTo(
N, PassThru, MGT->
getChain());
10767 return CombineTo(
N, NewLd, NewLd.
getValue(1));
10771 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
10783 EVT VT =
N->getValueType(0);
10784 if (!
Cond.hasOneUse() ||
Cond.getScalarValueSizeInBits() != 1 ||
10793 bool AllAddOne =
true;
10794 bool AllSubOne =
true;
10796 for (
unsigned i = 0; i != Elts; ++i) {
10815 if (AllAddOne || AllSubOne) {
10846 EVT VT =
N->getValueType(0);
10867 bool isAbs =
false;
10886 AddToWorklist(Shift.
getNode());
10887 AddToWorklist(
Add.getNode());
10914 EVT NarrowVT = LHS.getValueType();
10921 if (LHS.getOpcode() == ISD::LOAD && LHS.hasOneUse() &&
10922 SetCCWidth != 1 && SetCCWidth < WideWidth &&
10965 (OpLHS == CondLHS || OpRHS == CondLHS))
10968 if (OpRHS.getOpcode() == CondRHS.getOpcode() &&
10971 CondLHS == OpLHS) {
10976 return Cond->getAPIntValue() == ~Op->getAPIntValue();
11017 if (OpLHS == LHS) {
11033 return (!Op && !
Cond) ||
11035 Cond->getAPIntValue() == (-
Op->getAPIntValue() - 1));
11067 if (SimplifySelectOps(
N, N1, N2))
11087 if (
SDValue V = foldVSelectOfConstants(
N))
11115 AddToWorklist(
SCC.getNode());
11120 return SCCC->isZero() ? N3 : N2;
11124 if (
SCC->isUndef())
11131 SCC.getOperand(1), N2, N3,
SCC.getOperand(2));
11138 if (SimplifySelectOps(
N, N2, N3))
11142 return SimplifySelectCC(
SDLoc(
N), N0, N1, N2, N3, CC);
11150 N->
hasOneUse() &&
N->use_begin()->getOpcode() == ISD::BRCOND;
11153 EVT VT =
N->getValueType(0);
11173 bool Updated =
false;
11185 return True || False;
11189 if (!IsAlwaysTrueOrFalse(
Cond, N1C)) {
11207 SDLoc(
N), !PreferSetCC);
11215 SDValue NewSetCC = rebuildSetCC(Combined);
11249 if (!
N.hasOneUse())
11277 unsigned Opcode =
N->getOpcode();
11279 EVT VT =
N->getValueType(0);
11284 "Expected EXTEND dag node in input!");
11320 unsigned Opcode =
N->getOpcode();
11322 EVT VT =
N->getValueType(0);
11328 &&
"Expected EXTEND dag node in input!");
11334 return DAG.
getNode(Opcode,
DL, VT, N0);
11353 unsigned FoldOpc = Opcode;
11381 for (
unsigned i = 0; i != NumElts; ++i) {
11383 if (Op.isUndef()) {
11409 bool HasCopyToRegUses =
false;
11416 if (UI.getUse().getResNo() != N0.
getResNo())
11425 for (
unsigned i = 0; i != 2; ++i) {
11443 HasCopyToRegUses =
true;
11446 if (HasCopyToRegUses) {
11447 bool BothLiveOut =
false;
11452 BothLiveOut =
true;
11459 return ExtendNodes.
size();
11469 for (
SDNode *SetCC : SetCCs) {
11472 for (
unsigned j = 0; j != 2; ++j) {
11474 if (SOp == OrigLoad)
11488 EVT DstVT =
N->getValueType(0);
11493 "Unexpected node type (not an extend)!");
11531 EVT SplitSrcVT = SrcVT;
11532 EVT SplitDstVT = DstVT;
11545 const unsigned NumSplits =
11552 for (
unsigned Idx = 0; Idx < NumSplits; Idx++) {
11553 const unsigned Offset = Idx * Stride;
11571 AddToWorklist(NewChain.
getNode());
11573 CombineTo(
N, NewValue);
11579 ExtendSetCCUses(SetCCs, N0, NewValue, (
ISD::NodeType)
N->getOpcode());
11580 CombineTo(N0.
getNode(), Trunc, NewChain);
11588 EVT VT =
N->getValueType(0);
11589 EVT OrigVT =
N->getOperand(0).getValueType();
11612 EVT MemVT =
Load->getMemoryVT();
11633 Load->getChain(),
Load->getBasePtr(),
11634 Load->getMemoryVT(),
Load->getMemOperand());
11651 Load->getValueType(0), ExtLoad);
11652 CombineTo(Load, Trunc, ExtLoad.
getValue(1));
11656 recursivelyDeleteUnusedNodes(N0.
getNode());
11665SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(
SDNode *Cast) {
11666 unsigned CastOpcode = Cast->
getOpcode();
11668 CastOpcode ==
ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND ||
11670 "Unexpected opcode for vector select narrowing/widening");
11710 bool LegalOperations,
SDNode *
N,
11721 if ((LegalOperations || !LN0->
isSimple() ||
11732 Combiner.recursivelyDeleteUnusedNodes(LN0);
11754 bool DoXform =
true;
11767 Combiner.ExtendSetCCUses(SetCCs, N0, ExtLoad, ExtOpc);
11769 bool NoReplaceTrunc =
SDValue(LN0, 0).hasOneUse();
11771 if (NoReplaceTrunc) {
11773 Combiner.recursivelyDeleteUnusedNodes(LN0);
11811 bool LegalOperations) {
11823 EVT VT =
N->getValueType(0);
11824 EVT XVT =
X.getValueType();
11840 return DAG.
getNode(ShiftOpcode,
DL, VT, NotX, ShiftAmount);
11854 EVT VT =
N->getValueType(0);
11864 if (VT.
isVector() && !LegalOperations &&
11883 if (SVT == MatchingVecType) {
11899 auto IsFreeToExtend = [&](
SDValue V) {
11917 if (UI.getUse().getResNo() != 0 ||
User == N0.
getNode())
11922 if (
User->getOpcode() != ExtOpcode ||
User->getValueType(0) != VT)
11928 if (IsFreeToExtend(N00) && IsFreeToExtend(N01)) {
11931 return DAG.
getSetCC(
DL, VT, Ext0, Ext1, CC);
11948 SDValue ExtTrueVal = (SetCCWidth == 1)
11952 if (
SDValue SCC = SimplifySelectCC(
DL, N00, N01, ExtTrueVal, Zero, CC,
true))
11964 return DAG.
getSelect(
DL, VT, SetCC, ExtTrueVal, Zero);
11973 EVT VT =
N->getValueType(0);
11993 if (NarrowLoad.getNode() != N0.
getNode()) {
11994 CombineTo(N0.
getNode(), NarrowLoad);
11996 AddToWorklist(oye);
12004 unsigned OpBits =
Op.getScalarValueSizeInBits();
12009 if (OpBits == DestBits) {
12012 if (NumSignBits > DestBits-MidBits)
12014 }
else if (OpBits < DestBits) {
12017 if (NumSignBits > OpBits-MidBits)
12022 if (NumSignBits > OpBits-MidBits)
12029 if (OpBits < DestBits)
12031 else if (OpBits > DestBits)
12051 if (
SDValue ExtLoad = CombineExtLoad(
N))
12082 bool NoReplaceTruncAnd = !N0.
hasOneUse();
12083 bool NoReplaceTrunc =
SDValue(LN00, 0).hasOneUse();
12086 if (NoReplaceTruncAnd) {
12089 CombineTo(N0.
getNode(), TruncAnd);
12091 if (NoReplaceTrunc) {
12096 CombineTo(LN00, Trunc, ExtLoad.
getValue(1));
12114 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
12145 if (NewXor.getNode() == N0.
getNode()) {
12172 Op =
N->getOperand(0);
12178 N.getValueType().getScalarType() !=
MVT::i1 ||
12223 EVT VT =
N->getValueType(0);
12244 APInt TruncatedBits =
12246 APInt(
Op.getScalarValueSizeInBits(), 0) :
12247 APInt::getBitsSet(
Op.getScalarValueSizeInBits(),
12248 N0.getScalarValueSizeInBits(),
12249 std::
min(
Op.getScalarValueSizeInBits(),
12261 if (NarrowLoad.getNode() != N0.
getNode()) {
12262 CombineTo(N0.
getNode(), NarrowLoad);
12264 AddToWorklist(oye);
12279 AddToWorklist(
Op.getNode());
12283 return ZExtOrTrunc;
12289 AddToWorklist(
Op.getNode());
12327 if (
SDValue ExtLoad = CombineExtLoad(
N))
12343 bool DoXform =
true;
12350 if (isAndLoadExtLoad(AndC, LN00, LoadResultTy, ExtVT))
12367 bool NoReplaceTruncAnd = !N0.
hasOneUse();
12368 bool NoReplaceTrunc =
SDValue(LN00, 0).hasOneUse();
12371 if (NoReplaceTruncAnd) {
12374 CombineTo(N0.
getNode(), TruncAnd);
12376 if (NoReplaceTrunc) {
12381 CombineTo(LN00, Trunc, ExtLoad.
getValue(1));
12390 if (
SDValue ZExtLoad = CombineZExtLogicopShiftLoad(
N))
12406 if (!LegalOperations && VT.
isVector() &&
12440 if (
SDValue SCC = SimplifySelectCC(
12475 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
12489 EVT VT =
N->getValueType(0);
12511 if (NarrowLoad.getNode() != N0.
getNode()) {
12512 CombineTo(N0.
getNode(), NarrowLoad);
12514 AddToWorklist(oye);
12550 bool DoXform =
true;
12563 CombineTo(
N, ExtLoad);
12564 if (NoReplaceTrunc) {
12566 recursivelyDeleteUnusedNodes(LN0);
12570 CombineTo(LN0, Trunc, ExtLoad.
getValue(1));
12584 if (!LegalOperations || TLI.
isLoadExtLegal(ExtType, VT, MemVT)) {
12588 CombineTo(
N, ExtLoad);
12590 recursivelyDeleteUnusedNodes(LN0);
12604 if (VT.
isVector() && !LegalOperations) {
12632 if (
SDValue SCC = SimplifySelectCC(
12649 unsigned Opcode =
N->getOpcode();
12669 EVT MinAssertVT = AssertVT.
bitsLT(BigA_AssertVT) ? AssertVT : BigA_AssertVT;
12684 if (AssertVT.
bitsLT(BigA_AssertVT)) {
12705 std::max(AL, AAN->getAlign()));
12715 unsigned AlignShift =
Log2(AL);
12720 if (LHSAlignShift >= AlignShift || RHSAlignShift >= AlignShift) {
12721 if (LHSAlignShift < AlignShift)
12723 if (RHSAlignShift < AlignShift)
12738 unsigned Opc =
N->getOpcode();
12742 EVT VT =
N->getValueType(0);
12752 unsigned ShAmt = 0;
12757 bool HasShiftedOffset =
false;
12777 uint64_t MemoryWidth = LN->getMemoryVT().getScalarSizeInBits();
12778 if (MemoryWidth <= ShAmt)
12789 LN->getExtensionType() != ExtType)
12798 unsigned ActiveBits = 0;
12799 if (
Mask.isMask()) {
12800 ActiveBits =
Mask.countTrailingOnes();
12801 }
else if (
Mask.isShiftedMask(ShAmt, ActiveBits)) {
12802 HasShiftedOffset =
true;
12823 if (!
SRL.hasOneUse())
12836 ShAmt = SRL1C->getZExtValue();
12837 uint64_t MemoryWidth = LN->getMemoryVT().getSizeInBits();
12838 if (ShAmt >= MemoryWidth)
12866 const APInt& ShiftMask =
Mask->getConstantOperandAPInt(1);
12867 if (ShiftMask.
isMask()) {
12877 N0 =
SRL.getOperand(0);
12885 unsigned ShLeftAmt = 0;
12889 ShLeftAmt = N01->getZExtValue();
12902 !isLegalNarrowLdSt(LN0, ExtType, ExtVT, ShAmt))
12905 auto AdjustBigEndianShift = [&](
unsigned ShAmt) {
12906 unsigned LVTStoreBits =
12909 return LVTStoreBits - EVTStoreBits - ShAmt;
12914 unsigned PtrAdjustmentInBits =
12917 uint64_t PtrOff = PtrAdjustmentInBits / 8;
12925 AddToWorklist(NewPtr.
getNode());
12939 WorklistRemover DeadNodes(*
this);
12944 if (ShLeftAmt != 0) {
12959 if (HasShiftedOffset) {
12976 EVT VT =
N->getValueType(0);
13006 if ((N00Bits <= ExtVTBits ||
13024 if ((N00Bits == ExtVTBits ||
13025 (!IsZext && (N00Bits < ExtVTBits ||
13027 (!LegalOperations ||
13052 if (
SDValue NarrowLoad = reduceLoadWidth(
N))
13060 if (ShAmt->getAPIntValue().ule(VTBits - ExtVTBits)) {
13064 if (((VTBits - ExtVTBits) - ShAmt->getZExtValue()) < InSignBits)
13085 CombineTo(
N, ExtLoad);
13087 AddToWorklist(ExtLoad.
getNode());
13102 CombineTo(
N, ExtLoad);
13110 if (ExtVT == Ld->getMemoryVT() && N0.
hasOneUse() &&
13114 VT,
SDLoc(
N), Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(),
13115 Ld->getMask(), Ld->getPassThru(), ExtVT, Ld->getMemOperand(),
13116 Ld->getAddressingMode(),
ISD::SEXTLOAD, Ld->isExpandingLoad());
13117 CombineTo(
N, ExtMaskedLoad);
13126 ExtVT == GN0->getMemoryVT() &&
13128 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
13129 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
13135 CombineTo(
N, ExtLoad);
13137 AddToWorklist(ExtLoad.
getNode());
13154 EVT VT =
N->getValueType(0);
13171 EVT VT =
N->getValueType(0);
13186 if (
C.getNode() !=
N)
13236 EVT TrTy =
N->getValueType(0);
13240 auto NewEltCnt = EltCnt * SizeRatio;
13248 int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1));
13283 AddToWorklist(Amt.
getNode());
13320 if (BuildVectEltTy == TruncVecEltTy) {
13324 unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts;
13326 assert((BuildVecNumElts % TruncVecNumElts) == 0 &&
13327 "Invalid number of elements");
13330 for (
unsigned i = 0, e = BuildVecNumElts; i !=
e; i += TruncEltOffset)
13340 if (
SDValue Reduced = reduceLoadWidth(
N))
13364 unsigned NumDefs = 0;
13368 if (!
X.isUndef()) {
13385 if (NumDefs == 1) {
13386 assert(V.getNode() &&
"The single defined operand is empty!");
13388 for (
unsigned i = 0, e = VTs.
size(); i != e; ++i) {
13394 AddToWorklist(
NV.getNode());
13409 (!LegalOperations ||
13451 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
13465 if (!LegalOperations && N0.
hasOneUse() &&
13499 if (!LegalOperations && N0.
hasOneUse() &&
13535 !LD1->hasOneUse() || !LD2->hasOneUse() ||
13536 LD1->getAddressSpace() != LD2->getAddressSpace())
13539 bool LD1Fast =
false;
13540 EVT LD1VT = LD1->getValueType(0);
13545 *LD1->getMemOperand(), &LD1Fast) && LD1Fast)
13546 return DAG.
getLoad(VT,
SDLoc(
N), LD1->getChain(), LD1->getBasePtr(),
13547 LD1->getPointerInfo(), LD1->getAlign());
13562 EVT VT =
N->getValueType(0);
13577 FPOpcode = ISD::FABS;
13581 FPOpcode = ISD::FNEG;
13585 FPOpcode = ISD::FABS;
13599 LogicOp0.
getOpcode() == ISD::BITCAST &&
13602 NumFPLogicOpsConv++;
13613 EVT VT =
N->getValueType(0);
13630 return ConstantFoldBITCASTofBUILD_VECTOR(N0.
getNode(),
13638 if (!LegalOperations ||
13644 if (
C.getNode() !=
N)
13699 AddToWorklist(NewConv.
getNode());
13709 AddToWorklist(FlipBit.
getNode());
13716 AddToWorklist(
Hi.getNode());
13718 AddToWorklist(FlipBit.
getNode());
13722 AddToWorklist(FlipBits.
getNode());
13752 AddToWorklist(
X.getNode());
13756 if (OrigXWidth < VTWidth) {
13758 AddToWorklist(
X.getNode());
13759 }
else if (OrigXWidth > VTWidth) {
13764 X.getValueType(),
X,
13766 X.getValueType()));
13767 AddToWorklist(
X.getNode());
13769 AddToWorklist(
X.getNode());
13775 AddToWorklist(Cst.
getNode());
13777 AddToWorklist(
X.getNode());
13779 AddToWorklist(XorResult.
getNode());
13783 SDLoc(XorResult)));
13784 AddToWorklist(XorResult64.
getNode());
13788 AddToWorklist(FlipBit.
getNode());
13791 AddToWorklist(FlipBits.
getNode());
13797 AddToWorklist(
X.getNode());
13802 AddToWorklist(Cst.
getNode());
13810 if (
SDValue CombineLD = CombineConsecutiveLoads(N0.
getNode(), VT))
13825 auto PeekThroughBitcast = [&](
SDValue Op) {
13826 if (
Op.getOpcode() == ISD::BITCAST &&
13827 Op.getOperand(0).getValueType() == VT)
13846 for (
int i = 0; i != MaskScale; ++i)
13847 NewMask.
push_back(M < 0 ? -1 : M * MaskScale + i);
13852 return LegalShuffle;
13859 EVT VT =
N->getValueType(0);
13860 return CombineConsecutiveLoads(
N, VT);
13883ConstantFoldBITCASTofBUILD_VECTOR(
SDNode *BV,
EVT DstEltVT) {
13887 if (SrcEltVT == DstEltVT)
return SDValue(BV, 0);
13894 if (SrcBitSize == DstBitSize) {
13899 if (
Op.getValueType() != SrcEltVT)
13902 AddToWorklist(Ops.
back().getNode());
13916 BV = ConstantFoldBITCASTofBUILD_VECTOR(BV, IntVT).getNode();
13924 SDNode *Tmp = ConstantFoldBITCASTofBUILD_VECTOR(BV, TmpVT).getNode();
13927 return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT);
13941 if (!BVN->getConstantRawBits(IsLE, DstBitSize, RawBits, UndefElements))
13946 for (
unsigned I = 0,
E = RawBits.
size();
I !=
E; ++
I) {
13947 if (UndefElements[
I])
13963 N->getFlags().hasAllowContract();
13968 return Options.NoInfsFPMath ||
N->getFlags().hasNoInfs();
13975 EVT VT =
N->getValueType(0);
13981 bool HasFMAD = (LegalOperations && TLI.
isFMADLegal(DAG,
N));
13989 if (!HasFMAD && !HasFMA)
13992 bool CanReassociate =
13993 Options.UnsafeFPMath ||
N->getFlags().hasAllowReassociation();
13995 Options.UnsafeFPMath || HasFMAD);
13997 if (!AllowFusionGlobally && !
N->getFlags().hasAllowContract())
14008 unsigned Opcode =
N.getOpcode();
14017 return AllowFusionGlobally ||
N->getFlags().hasAllowContract();
14043 if (CanReassociate && isFusedOp(N0) &&
14048 }
else if (CanReassociate && isFusedOp(N1) &&
14060 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
A,
B, CDE);
14066 if (N0.
getOpcode() == ISD::FP_EXTEND) {
14071 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
14080 if (N1.
getOpcode() == ISD::FP_EXTEND) {
14085 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
14098 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X,
Y,
14099 DAG.
getNode(PreferredFusedOpcode, SL, VT,
14100 DAG.
getNode(ISD::FP_EXTEND, SL, VT, U),
14101 DAG.
getNode(ISD::FP_EXTEND, SL, VT, V),
14104 if (isFusedOp(N0)) {
14106 if (N02.
getOpcode() == ISD::FP_EXTEND) {
14126 PreferredFusedOpcode, SL, VT, DAG.
getNode(ISD::FP_EXTEND, SL, VT,
X),
14127 DAG.
getNode(ISD::FP_EXTEND, SL, VT,
Y),
14128 DAG.
getNode(PreferredFusedOpcode, SL, VT,
14129 DAG.
getNode(ISD::FP_EXTEND, SL, VT, U),
14130 DAG.
getNode(ISD::FP_EXTEND, SL, VT, V),
Z));
14132 if (N0.
getOpcode() == ISD::FP_EXTEND) {
14134 if (isFusedOp(N00)) {
14148 if (isFusedOp(N1)) {
14150 if (N12.
getOpcode() == ISD::FP_EXTEND) {
14167 if (N1.
getOpcode() == ISD::FP_EXTEND) {
14169 if (isFusedOp(N10)) {
14189 EVT VT =
N->getValueType(0);
14194 bool HasFMAD = (LegalOperations && TLI.
isFMADLegal(DAG,
N));
14202 if (!HasFMAD && !HasFMA)
14207 Options.UnsafeFPMath || HasFMAD);
14210 if (!AllowFusionGlobally && !
N->getFlags().hasAllowContract())
14219 bool NoSignedZero =
Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros();
14226 return AllowFusionGlobally ||
N->getFlags().hasAllowContract();
14242 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
14243 DAG.
getNode(ISD::FNEG, SL, VT, YZ.getOperand(0)),
14254 if (
SDValue V = tryToFoldXSubYZ(N0, N1))
14257 if (
SDValue V = tryToFoldXYSubZ(N0, N1))
14261 if (
SDValue V = tryToFoldXYSubZ(N0, N1))
14264 if (
SDValue V = tryToFoldXSubYZ(N0, N1))
14273 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
14274 DAG.
getNode(ISD::FNEG, SL, VT, N00), N01,
14275 DAG.
getNode(ISD::FNEG, SL, VT, N1));
14282 if (N0.
getOpcode() == ISD::FP_EXTEND) {
14287 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
14290 DAG.
getNode(ISD::FNEG, SL, VT, N1));
14297 if (N1.
getOpcode() == ISD::FP_EXTEND) {
14303 PreferredFusedOpcode, SL, VT,
14304 DAG.
getNode(ISD::FNEG, SL, VT,
14316 if (N0.
getOpcode() == ISD::FP_EXTEND) {
14325 DAG.
getNode(PreferredFusedOpcode, SL, VT,
14341 if (N00.
getOpcode() == ISD::FP_EXTEND) {
14348 DAG.
getNode(PreferredFusedOpcode, SL, VT,
14357 return Options.UnsafeFPMath ||
N->getFlags().hasAllowReassociation();
14366 unsigned Opcode =
N.getOpcode();
14372 bool CanFuse =
Options.UnsafeFPMath ||
N->getFlags().hasAllowContract();
14375 if (CanFuse && isFusedOp(N0) &&
14376 isContractableAndReassociableFMUL(N0.
getOperand(2)) &&
14380 DAG.
getNode(PreferredFusedOpcode, SL, VT,
14383 DAG.
getNode(ISD::FNEG, SL, VT, N1)));
14388 if (CanFuse && isFusedOp(N1) &&
14389 isContractableAndReassociableFMUL(N1.
getOperand(2)) &&
14394 PreferredFusedOpcode, SL, VT,
14396 DAG.
getNode(PreferredFusedOpcode, SL, VT,
14397 DAG.
getNode(ISD::FNEG, SL, VT, N20), N21, N0));
14402 if (isFusedOp(N0) && N0->
hasOneUse()) {
14404 if (N02.
getOpcode() == ISD::FP_EXTEND) {
14406 if (isContractableAndReassociableFMUL(N020) &&
14412 PreferredFusedOpcode, SL, VT,
14415 DAG.
getNode(ISD::FNEG, SL, VT, N1)));
14426 if (N0.
getOpcode() == ISD::FP_EXTEND) {
14428 if (isFusedOp(N00)) {
14430 if (isContractableAndReassociableFMUL(N002) &&
14434 PreferredFusedOpcode, SL, VT,
14438 PreferredFusedOpcode, SL, VT,
14441 DAG.
getNode(ISD::FNEG, SL, VT, N1)));
14451 if (isContractableAndReassociableFMUL(N120) &&
14457 PreferredFusedOpcode, SL, VT,
14459 DAG.
getNode(PreferredFusedOpcode, SL, VT,
14460 DAG.
getNode(ISD::FNEG, SL, VT,
14461 DAG.
getNode(ISD::FP_EXTEND, SL, VT, N1200)),
14462 DAG.
getNode(ISD::FP_EXTEND, SL, VT, N1201), N0));
14477 if (isContractableAndReassociableFMUL(N102) &&
14483 PreferredFusedOpcode, SL, VT,
14484 DAG.
getNode(ISD::FNEG, SL, VT,
14485 DAG.
getNode(ISD::FP_EXTEND, SL, VT, N100)),
14486 DAG.
getNode(ISD::FP_EXTEND, SL, VT, N101),
14487 DAG.
getNode(PreferredFusedOpcode, SL, VT,
14488 DAG.
getNode(ISD::FNEG, SL, VT,
14489 DAG.
getNode(ISD::FP_EXTEND, SL, VT, N1020)),
14490 DAG.
getNode(ISD::FP_EXTEND, SL, VT, N1021), N0));
14501SDValue DAGCombiner::visitFMULForFMADistributiveCombine(
SDNode *
N) {
14504 EVT VT =
N->getValueType(0);
14525 bool HasFMAD =
Options.UnsafeFPMath &&
14529 if (!HasFMAD && !HasFMA)
14541 if (
C->isExactlyValue(+1.0))
14542 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
14544 if (
C->isExactlyValue(-1.0))
14545 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
14546 DAG.
getNode(ISD::FNEG, SL, VT,
Y));
14552 if (
SDValue FMA = FuseFADD(N0, N1))
14554 if (
SDValue FMA = FuseFADD(N1, N0))
14564 if (C0->isExactlyValue(+1.0))
14565 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
14566 DAG.
getNode(ISD::FNEG, SL, VT,
X.getOperand(1)),
Y,
14568 if (C0->isExactlyValue(-1.0))
14569 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
14570 DAG.
getNode(ISD::FNEG, SL, VT,
X.getOperand(1)),
Y,
14571 DAG.
getNode(ISD::FNEG, SL, VT,
Y));
14574 if (C1->isExactlyValue(+1.0))
14575 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
14576 DAG.
getNode(ISD::FNEG, SL, VT,
Y));
14577 if (C1->isExactlyValue(-1.0))
14578 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
14585 if (
SDValue FMA = FuseFSUB(N0, N1))
14587 if (
SDValue FMA = FuseFSUB(N1, N0))
14598 EVT VT =
N->getValueType(0);
14612 if (N0CFP && !N1CFP)
14617 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
14622 if (N1C && N1C->
isZero())
14623 if (N1C->
isNegative() ||
Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros())
14626 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
14632 N1, DAG, LegalOperations, ForCodeSize))
14638 N0, DAG, LegalOperations, ForCodeSize))
14645 return C &&
C->isExactlyValue(-2.0);
14649 if (isFMulNegTwo(N0)) {
14655 if (isFMulNegTwo(N1)) {
14666 if ((
Options.NoNaNsFPMath || Flags.hasNoNaNs()) && AllowNewConst) {
14680 (Flags.hasAllowReassociation() && Flags.hasNoSignedZeros())) &&
14698 if (CFP01 && !CFP00 && N0.
getOperand(0) == N1) {
14719 if (CFP11 && !CFP10 && N1.
getOperand(0) == N0) {
14767 if (
SDValue Fused = visitFADDForFMACombine(
N)) {
14768 AddToWorklist(Fused.getNode());
14778 EVT VT =
N->getValueType(0);
14779 EVT ChainVT =
N->getValueType(1);
14786 N1, DAG, LegalOperations, ForCodeSize)) {
14788 {Chain, N0, NegN1});
14794 N0, DAG, LegalOperations, ForCodeSize)) {
14796 {Chain, N1, NegN0});
14806 EVT VT =
N->getValueType(0);
14821 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
14824 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
14828 if (N1CFP && N1CFP->
isZero()) {
14830 Flags.hasNoSignedZeros()) {
14837 if (
Options.NoNaNsFPMath || Flags.hasNoNaNs())
14842 if (N0CFP && N0CFP->
isZero()) {
14844 (
Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros())) {
14855 return DAG.
getNode(ISD::FNEG,
DL, VT, N1);
14861 (Flags.hasAllowReassociation() && Flags.hasNoSignedZeros())) &&
14877 if (
SDValue Fused = visitFSUBForFMACombine(
N)) {
14878 AddToWorklist(Fused.getNode());
14889 EVT VT =
N->getValueType(0);
14909 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
14912 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
14915 if (
Options.UnsafeFPMath || Flags.hasAllowReassociation()) {
14961 if (NegN0 && NegN1 &&
14968 if (Flags.hasNoNaNs() && Flags.hasNoSignedZeros() &&
14979 if (TrueOpnd && FalseOpnd &&
15000 if (TrueOpnd->isExactlyValue(-1.0) && FalseOpnd->isExactlyValue(1.0) &&
15004 if (TrueOpnd->isExactlyValue(1.0) && FalseOpnd->isExactlyValue(-1.0))
15013 if (
SDValue Fused = visitFMULForFMADistributiveCombine(
N)) {
15014 AddToWorklist(Fused.getNode());
15027 EVT VT =
N->getValueType(0);
15033 bool CanReassociate =
15034 Options.UnsafeFPMath ||
N->getFlags().hasAllowReassociation();
15052 if (NegN0 && NegN1 &&
15059 if (N0CFP && N0CFP->
isZero())
15061 if (N1CFP && N1CFP->
isZero())
15075 if (CanReassociate) {
15102 AddToWorklist(RHSNeg.
getNode());
15112 DAG.
getNode(ISD::FNEG,
DL, VT, N1), N2);
15116 if (CanReassociate) {
15118 if (N1CFP && N0 == N2) {
15136 SDValue(
N, 0), DAG, LegalOperations, ForCodeSize))
15137 return DAG.
getNode(ISD::FNEG,
DL, VT, Neg);
15154 if (LegalDAG || (!UnsafeMath && !Flags.hasAllowReciprocal()))
15169 unsigned NumElts = 1;
15170 EVT VT =
N->getValueType(0);
15174 if (!MinUses || (N1->
use_size() * NumElts) < MinUses)
15180 for (
auto *U : N1->
uses()) {
15181 if (U->getOpcode() ==
ISD::FDIV && U->getOperand(1) == N1) {
15183 if (U->getOperand(1).getOpcode() == ISD::FSQRT &&
15184 U->getOperand(0) == U->getOperand(1).getOperand(0) &&
15185 U->getFlags().hasAllowReassociation() &&
15186 U->getFlags().hasNoSignedZeros())
15191 if (UnsafeMath || U->getFlags().hasAllowReciprocal())
15198 if ((
Users.size() * NumElts) < MinUses)
15206 for (
auto *U :
Users) {
15208 if (Dividend != FPOne) {
15210 Reciprocal, Flags);
15211 CombineTo(U, NewNode);
15212 }
else if (U != Reciprocal.
getNode()) {
15215 CombineTo(U, Reciprocal);
15224 EVT VT =
N->getValueType(0);
15239 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
15242 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
15248 if (
Options.UnsafeFPMath || Flags.hasAllowReciprocal()) {
15258 (!LegalOperations ||
15273 }
else if (N1.
getOpcode() == ISD::FP_EXTEND &&
15303 if (Flags.hasAllowReassociation() && N1.
hasOneUse() &&
15306 if (
Y.getOpcode() == ISD::FABS &&
Y.hasOneUse())
15316 if (
SDValue Rsqrt = buildRsqrtEstimate(AAZ, Flags))
15320 recursivelyDeleteUnusedNodes(AAZ.
getNode());
15328 AddToWorklist(Div.
getNode());
15335 if (
Options.NoInfsFPMath || Flags.hasNoInfs())
15336 if (
SDValue RV = BuildDivEstimate(N0, N1, Flags))
15341 if ((
Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) &&
15342 (
Options.UnsafeFPMath || Flags.hasAllowReassociation()))
15355 if (NegN0 && NegN1 &&
15366 EVT VT =
N->getValueType(0);
15377 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
15389 if (!Flags.hasApproximateFuncs() ||
15390 (!
Options.NoInfsFPMath && !Flags.hasNoInfs()))
15402 return buildSqrtEstimate(N0, Flags);
15409 if ((N1.
getOpcode() == ISD::FP_EXTEND ||
15415 if (N1VT == N1Op0VT)
15437 EVT VT =
N->getValueType(0);
15448 if (!V.isNegative()) {
15491 EVT VT =
N->getValueType(0);
15501 if (!Flags.hasNoSignedZeros() || !Flags.hasNoInfs() || !Flags.hasNoNaNs() ||
15502 !Flags.hasApproximateFuncs())
15521 if (ExponentIs025 || ExponentIs075) {
15532 if ((!Flags.hasNoSignedZeros() && ExponentIs025) || !Flags.hasNoInfs() ||
15533 !Flags.hasApproximateFuncs())
15566 EVT VT =
N->getValueType(0);
15587 EVT VT =
N->getValueType(0);
15597 (!LegalOperations ||
15639 EVT VT =
N->getValueType(0);
15649 (!LegalOperations ||
15679 EVT VT =
N->getValueType(0);
15685 EVT SrcVT = Src.getValueType();
15700 unsigned ActualSize = std::min(InputSize, OutputSize);
15720 EVT VT =
N->getValueType(0);
15735 EVT VT =
N->getValueType(0);
15752 EVT VT =
N->getValueType(0);
15764 const bool NIsTrunc =
N->getConstantOperandVal(1) == 1;
15793 AddToWorklist(Tmp.
getNode());
15798 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
15806 EVT VT =
N->getValueType(0);
15809 if (
N->hasOneUse() &&
15818 if (N0.
getOpcode() == ISD::FP16_TO_FP &&
15827 if (
In.getValueType() == VT)
return In;
15842 CombineTo(
N, ExtLoad);
15851 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
15859 EVT VT =
N->getValueType(0);
15870 EVT VT =
N->getValueType(0);
15883 case ISD::FNEARBYINT:
15894 EVT VT =
N->getValueType(0);
15905 EVT VT =
N->getValueType(0);
15922 N->getFlags().hasNoSignedZeros()) && N0.
hasOneUse()) {
15927 if (
SDValue Cast = foldSignChangeInBitcast(
N))
15936 EVT VT =
N->getValueType(0);
15938 unsigned Opc =
N->getOpcode();
15939 bool PropagatesNaN = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM;
15940 bool IsMin = Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM;
15960 return PropagatesNaN ?
N->getOperand(1) :
N->getOperand(0);
15969 if (IsMin == AF.
isNegative() && (!PropagatesNaN || Flags.hasNoNaNs()))
15970 return N->getOperand(1);
15976 if (IsMin != AF.
isNegative() && (PropagatesNaN || Flags.hasNoNaNs()))
15977 return N->getOperand(0);
15986 EVT VT =
N->getValueType(0);
16001 if (
SDValue Cast = foldSignChangeInBitcast(
N))
16039 if (
SDValue NewN1 = rebuildSetCC(N1))
16041 ChainHandle.getValue(), NewN1, N2);
16050 (
N.getOperand(0).hasOneUse() &&
16051 N.getOperand(0).getOpcode() ==
ISD::SRL))) {
16054 N =
N.getOperand(0);
16103 SDValue Tmp = visitXOR(
N.getNode());
16110 N = XORHandle.getValue();
16122 bool Equal =
false;
16132 EVT SetCCVT =
N.getValueType();
16173 bool &IsLoad,
bool &IsMasked,
SDValue &Ptr,
16176 if (LD->isIndexed())
16178 EVT VT = LD->getMemoryVT();
16181 Ptr = LD->getBasePtr();
16183 if (ST->isIndexed())
16185 EVT VT = ST->getMemoryVT();
16188 Ptr = ST->getBasePtr();
16191 if (LD->isIndexed())
16193 EVT VT = LD->getMemoryVT();
16197 Ptr = LD->getBasePtr();
16200 if (ST->isIndexed())
16202 EVT VT = ST->getMemoryVT();
16206 Ptr = ST->getBasePtr();
16220bool DAGCombiner::CombineToPreIndexedLoadStore(
SDNode *
N) {
16224 bool IsLoad =
true;
16225 bool IsMasked =
false;
16247 bool Swapped =
false;
16276 if (Val == BasePtr)
16331 bool RealUse =
false;
16367 WorklistRemover DeadNodes(*
this);
16376 deleteAndRecombine(
N);
16382 for (
unsigned i = 0, e = OtherUses.
size(); i != e; ++i) {
16383 unsigned OffsetIdx = 1;
16384 if (OtherUses[i]->getOperand(OffsetIdx).getNode() ==
BasePtr.getNode())
16386 assert(OtherUses[i]->getOperand(!OffsetIdx).getNode() ==
16387 BasePtr.getNode() &&
"Expected BasePtr operand");
16401 const APInt &Offset0 = CN->getAPIntValue();
16403 int X0 = (OtherUses[i]->getOpcode() ==
ISD::SUB && OffsetIdx == 1) ? -1 : 1;
16404 int Y0 = (OtherUses[i]->getOpcode() ==
ISD::SUB && OffsetIdx == 0) ? -1 : 1;
16410 APInt CNV = Offset0;
16411 if (X0 < 0) CNV = -CNV;
16412 if (X1 * Y0 * Y1 < 0) CNV = CNV + Offset1;
16413 else CNV = CNV - Offset1;
16425 deleteAndRecombine(OtherUses[i]);
16430 deleteAndRecombine(Ptr.
getNode());
16431 AddToWorklist(
Result.getNode());
16462 bool IsLoad =
true;
16463 bool IsMasked =
false;
16466 IsMasked, OtherPtr, TLI)) {
16486 bool &IsMasked,
SDValue &Ptr,
16492 IsMasked, Ptr, TLI) ||
16525bool DAGCombiner::CombineToPostIndexedLoadStore(
SDNode *
N) {
16529 bool IsLoad =
true;
16530 bool IsMasked =
false;
16551 ++PostIndexedNodes;
16555 WorklistRemover DeadNodes(*
this);
16564 deleteAndRecombine(
N);
16568 Result.getValue(IsLoad ? 1 : 0));
16569 deleteAndRecombine(Op);
16585 "Cannot split out indexing using opaque target constants");
16602 Val =
ST->getValue();
16604 EVT STMemType =
ST->getMemoryVT();
16605 if (STType == STMemType)
16611 Val = DAG.
getNode(ISD::FTRUNC,
SDLoc(ST), STMemType, Val);
16627 EVT LDMemType =
LD->getMemoryVT();
16628 EVT LDType =
LD->getValueType(0);
16630 "Attempting to extend value of non-matching type");
16631 if (LDType == LDMemType)
16634 switch (
LD->getExtensionType()) {
16658 if (!ST || !
ST->isSimple())
16661 EVT LDType =
LD->getValueType(0);
16662 EVT LDMemType =
LD->getMemoryVT();
16663 EVT STMemType =
ST->getMemoryVT();
16664 EVT STType =
ST->getValue().getValueType();
16687 if (!BasePtrST.equalBaseIndex(BasePtrLD, DAG,
Offset))
16706 STCoversLD = (
Offset == 0) && LdMemSize == StMemSize;
16712 if (
LD->isIndexed()) {
16717 SDValue Idx = SplitIndexingFromLoad(LD);
16718 SDValue Ops[] = {Val, Idx, Chain};
16719 return CombineTo(LD, Ops, 3);
16721 return CombineTo(LD, Val, Chain);
16728 if (
Offset == 0 && LDType == STType && STMemType == LDMemType) {
16731 return ReplaceLd(LD,
ST->getValue(), Chain);
16739 SDLoc(ST), STType);
16741 return ReplaceLd(LD, Val, Chain);
16746 if (
LD->getBasePtr().isUndef() ||
Offset != 0)
16752 if (!getTruncatedStoreValue(ST, Val))
16756 if (STMemType != LDMemType) {
16764 if (!extendLoadedValueToExtension(LD, Val))
16766 return ReplaceLd(LD, Val, Chain);
16771 deleteAndRecombine(Val.
getNode());
16784 if (
LD->isSimple()) {
16787 if (!
N->hasAnyUseOfValue(0)) {
16795 dbgs() <<
"\nWith chain: "; Chain.dump(&DAG);
16797 WorklistRemover DeadNodes(*
this);
16799 AddUsersToWorklist(Chain.getNode());
16800 if (
N->use_empty())
16801 deleteAndRecombine(
N);
16815 if (!
N->hasAnyUseOfValue(0) && (CanSplitIdx || !
N->hasAnyUseOfValue(1))) {
16818 if (
N->hasAnyUseOfValue(1) && CanSplitIdx) {
16819 Index = SplitIndexingFromLoad(LD);
16822 AddUsersToWorklist(
N);
16826 dbgs() <<
"\nWith: ";
Undef.dump(&DAG);
16827 dbgs() <<
" and 2 other values\n");
16828 WorklistRemover DeadNodes(*
this);
16832 deleteAndRecombine(
N);
16840 if (
auto V = ForwardStoreValueToDirectLoad(LD))
16846 if (*Alignment >
LD->getAlign() &&
16847 isAligned(*Alignment,
LD->getSrcValueOffset())) {
16849 LD->getExtensionType(),
SDLoc(
N),
LD->getValueType(0), Chain, Ptr,
16850 LD->getPointerInfo(),
LD->getMemoryVT(), *Alignment,
16851 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
16859 if (
LD->isUnindexed()) {
16861 SDValue BetterChain = FindBetterChain(LD, Chain);
16864 if (Chain != BetterChain) {
16870 BetterChain, Ptr,
LD->getMemOperand());
16873 LD->getValueType(0),
16874 BetterChain, Ptr,
LD->getMemoryVT(),
16875 LD->getMemOperand());
16883 return CombineTo(
N, ReplLoad.
getValue(0), Token);
16888 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
16893 if (SliceUpLoad(
N))
16913struct LoadedSlice {
16917 bool ForCodeSize =
false;
16920 unsigned Loads = 0;
16921 unsigned Truncates = 0;
16922 unsigned CrossRegisterBanksCopies = 0;
16923 unsigned ZExts = 0;
16924 unsigned Shift = 0;
16926 explicit Cost(
bool ForCodeSize) : ForCodeSize(ForCodeSize) {}
16929 Cost(
const LoadedSlice &LS,
bool ForCodeSize)
16930 : ForCodeSize(ForCodeSize), Loads(1) {
16931 EVT TruncType =
LS.Inst->getValueType(0);
16932 EVT LoadedType =
LS.getLoadedType();
16933 if (TruncType != LoadedType &&
16934 !
LS.DAG->getTargetLoweringInfo().isZExtFree(LoadedType, TruncType))
16942 void addSliceGain(
const LoadedSlice &LS) {
16946 LS.Inst->getValueType(0)))
16952 if (
LS.canMergeExpensiveCrossRegisterBankCopy())
16953 ++CrossRegisterBanksCopies;
16957 Loads += RHS.Loads;
16958 Truncates += RHS.Truncates;
16959 CrossRegisterBanksCopies += RHS.CrossRegisterBanksCopies;
16960 ZExts += RHS.ZExts;
16961 Shift += RHS.Shift;
16966 return Loads == RHS.Loads && Truncates == RHS.Truncates &&
16967 CrossRegisterBanksCopies == RHS.CrossRegisterBanksCopies &&
16968 ZExts == RHS.ZExts && Shift == RHS.Shift;
16971 bool operator!=(
const Cost &RHS)
const {
return !(*
this ==
RHS); }
16973 bool operator<(
const Cost &RHS)
const {
16976 unsigned ExpensiveOpsLHS = Loads + CrossRegisterBanksCopies;
16977 unsigned ExpensiveOpsRHS = RHS.Loads + RHS.CrossRegisterBanksCopies;
16980 if (!ForCodeSize && ExpensiveOpsLHS != ExpensiveOpsRHS)
16981 return ExpensiveOpsLHS < ExpensiveOpsRHS;
16982 return (Truncates + ZExts + Shift + ExpensiveOpsLHS) <
16983 (RHS.Truncates + RHS.ZExts + RHS.Shift + ExpensiveOpsRHS);
16986 bool operator>(
const Cost &RHS)
const {
return RHS < *
this; }
16988 bool operator<=(
const Cost &RHS)
const {
return !(RHS < *
this); }
16990 bool operator>=(
const Cost &RHS)
const {
return !(*
this <
RHS); }
17009 : Inst(Inst), Origin(Origin), Shift(Shift), DAG(DAG) {}
17014 APInt getUsedBits()
const {
17019 assert(Origin &&
"No original load to compare against.");
17021 assert(Inst &&
"This slice is not bound to an instruction");
17023 "Extracted slice is bigger than the whole type!");
17025 UsedBits.setAllBits();
17026 UsedBits = UsedBits.zext(
BitWidth);
17027 UsedBits <<= Shift;
17032 unsigned getLoadedSize()
const {
17033 unsigned SliceSize = getUsedBits().countPopulation();
17034 assert(!(SliceSize & 0x7) &&
"Size is not a multiple of a byte.");
17035 return SliceSize / 8;
17040 EVT getLoadedType()
const {
17041 assert(DAG &&
"Missing context");
17056 bool isLegal()
const {
17058 if (!Origin || !Inst || !DAG)
17068 EVT SliceType = getLoadedType();
17092 if (TruncateType != SliceType &&
17102 uint64_t getOffsetFromBase()
const {
17103 assert(DAG &&
"Missing context.");
17105 assert(!(Shift & 0x7) &&
"Shifts not aligned on Bytes are not supported.");
17109 "The size of the original loaded type is not a multiple of a"
17114 "Invalid shift amount for given loaded size");
17127 assert(Inst && Origin &&
"Unable to replace a non-existing slice.");
17129 SDValue BaseAddr = OldBaseAddr;
17131 int64_t
Offset =
static_cast<int64_t
>(getOffsetFromBase());
17132 assert(
Offset >= 0 &&
"Offset too big to fit in int64_t!");
17142 EVT SliceType = getLoadedType();
17152 if (SliceType != FinalType)
17162 bool canMergeExpensiveCrossRegisterBankCopy()
const {
17166 if (
Use->getOpcode() != ISD::BITCAST)
17168 assert(DAG &&
"Missing context");
17170 EVT ResVT =
Use->getValueType(0);
17175 Use->getOperand(0)->isDivergent());
17184 if (!
TRI ||
TRI->getCommonSubClass(ArgRC, ResRC))
17189 bool IsFast =
false;
17230 const LoadedSlice &Second) {
17231 assert(First.Origin == Second.Origin && First.Origin &&
17232 "Unable to match different memory origins.");
17233 APInt UsedBits = First.getUsedBits();
17234 assert((UsedBits & Second.getUsedBits()) == 0 &&
17235 "Slices are not supposed to overlap.");
17236 UsedBits |= Second.getUsedBits();
17245 LoadedSlice::Cost &GlobalLSCost) {
17246 unsigned NumberOfSlices = LoadedSlices.
size();
17248 if (NumberOfSlices < 2)
17253 llvm::sort(LoadedSlices, [](
const LoadedSlice &LHS,
const LoadedSlice &RHS) {
17254 assert(LHS.Origin == RHS.Origin &&
"Different bases not implemented.");
17255 return LHS.getOffsetFromBase() < RHS.getOffsetFromBase();
17257 const TargetLowering &TLI = LoadedSlices[0].DAG->getTargetLoweringInfo();
17260 const LoadedSlice *First =
nullptr;
17261 const LoadedSlice *Second =
nullptr;
17262 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice,
17265 Second = &LoadedSlices[CurrSlice];
17272 EVT LoadedType = First->getLoadedType();
17275 if (LoadedType != Second->getLoadedType())
17279 Align RequiredAlignment;
17280 if (!TLI.hasPairedLoad(LoadedType, RequiredAlignment)) {
17286 if (First->getAlign() < RequiredAlignment)
17293 assert(GlobalLSCost.Loads > 0 &&
"We save more loads than we created!");
17294 --GlobalLSCost.Loads;
17311 const APInt &UsedBits,
bool ForCodeSize) {
17312 unsigned NumberOfSlices = LoadedSlices.
size();
17314 return NumberOfSlices > 1;
17317 if (NumberOfSlices != 2)
17325 LoadedSlice::Cost OrigCost(ForCodeSize), GlobalSlicingCost(ForCodeSize);
17327 OrigCost.Loads = 1;
17328 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice) {
17329 const LoadedSlice &LS = LoadedSlices[CurrSlice];
17331 LoadedSlice::Cost SliceCost(LS, ForCodeSize);
17332 GlobalSlicingCost += SliceCost;
17336 OrigCost.addSliceGain(LS);
17341 return OrigCost > GlobalSlicingCost;
17350bool DAGCombiner::SliceUpLoad(
SDNode *
N) {
17356 !
LD->getValueType(0).isInteger())
17362 if (
LD->getValueType(0).isScalableVector())
17367 APInt UsedBits(
LD->getValueSizeInBits(0), 0);
17375 UI != UIEnd; ++UI) {
17377 if (UI.getUse().getResNo() != 0)
17381 unsigned Shift = 0;
17386 Shift =
User->getConstantOperandVal(1);
17399 unsigned Width =
User->getValueSizeInBits(0);
17404 LoadedSlice
LS(
User, LD, Shift, &DAG);
17405 APInt CurrentUsedBits =
LS.getUsedBits();
17408 if ((CurrentUsedBits & UsedBits) != 0)
17411 UsedBits |= CurrentUsedBits;
17432 for (
const LoadedSlice &LS : LoadedSlices) {
17434 CombineTo(
LS.Inst, SliceInst,
true);
17435 if (SliceInst.
getOpcode() != ISD::LOAD)
17438 "It takes more than a zext to get to the loaded slice!!");
17445 AddToWorklist(Chain.getNode());
17452static std::pair<unsigned, unsigned>
17454 std::pair<unsigned, unsigned> Result(0, 0);
17464 if (LD->getBasePtr() != Ptr)
return Result;
17467 if (V.getValueType() !=
MVT::i16 &&
17475 uint64_t NotMask = ~cast<ConstantSDNode>(V->getOperand(1))->getSExtValue();
17477 if (NotMaskLZ & 7)
return Result;
17479 if (NotMaskTZ & 7)
return Result;
17480 if (NotMaskLZ == 64)
return Result;
17487 if (V.getValueType() !=
MVT::i64 && NotMaskLZ)
17488 NotMaskLZ -= 64-V.getValueSizeInBits();
17490 unsigned MaskedBytes = (V.getValueSizeInBits()-NotMaskLZ-NotMaskTZ)/8;
17491 switch (MaskedBytes) {
17495 default:
return Result;
17500 if (NotMaskTZ && NotMaskTZ/8 % MaskedBytes)
return Result;
17504 if (LD == Chain.getNode())
17509 if (!LD->isOperandOf(Chain.getNode()))
17514 Result.first = MaskedBytes;
17515 Result.second = NotMaskTZ/8;
17526 unsigned NumBytes = MaskInfo.first;
17527 unsigned ByteShift = MaskInfo.second;
17533 ByteShift*8, (ByteShift+NumBytes)*8);
17542 bool UseTruncStore;
17543 if (
DC->isTypeLegal(VT))
17544 UseTruncStore =
false;
17547 UseTruncStore =
true;
17568 StOffset = ByteShift;
17599 if (!
ST->isSimple())
17607 if (
ST->isTruncatingStore() || VT.
isVector())
17610 unsigned Opc =
Value.getOpcode();
17622 std::pair<unsigned, unsigned> MaskedLoad;
17624 if (MaskedLoad.first)
17626 Value.getOperand(1), ST,
this))
17631 if (MaskedLoad.first)
17633 Value.getOperand(0), ST,
this))
17647 if (
LD->getBasePtr() != Ptr ||
17648 LD->getPointerInfo().getAddrSpace() !=
17649 ST->getPointerInfo().getAddrSpace())
17658 if (Imm == 0 ||
Imm.isAllOnes())
17660 unsigned ShAmt =
Imm.countTrailingZeros();
17661 unsigned MSB =
BitWidth -
Imm.countLeadingZeros() - 1;
17679 ShAmt = (((ShAmt + NewBW - 1) / NewBW) * NewBW) - NewBW;
17681 std::min(
BitWidth, ShAmt + NewBW));
17682 if ((Imm & Mask) == Imm) {
17690 PtrOff = (
BitWidth + 7 - NewBW) / 8 - PtrOff;
17692 bool IsFast =
false;
17695 LD->getAddressSpace(), NewAlign,
17696 LD->getMemOperand()->getFlags(), &IsFast) ||
17704 LD->getPointerInfo().getWithOffset(PtrOff), NewAlign,
17705 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
17711 ST->getPointerInfo().getWithOffset(PtrOff), NewAlign);
17713 AddToWorklist(NewPtr.
getNode());
17714 AddToWorklist(NewLD.
getNode());
17715 AddToWorklist(NewVal.
getNode());
17716 WorklistRemover DeadNodes(*
this);
17735 EVT VT =
LD->getMemoryVT();
17737 VT !=
ST->getMemoryVT() ||
17738 LD->isNonTemporal() ||
17739 ST->isNonTemporal() ||
17740 LD->getPointerInfo().getAddrSpace() != 0 ||
17741 ST->getPointerInfo().getAddrSpace() != 0)
17751 bool FastLD =
false, FastST =
false;
17758 *
LD->getMemOperand(), &FastLD) ||
17760 *
ST->getMemOperand(), &FastST) ||
17761 !FastLD || !FastST)
17766 LD->getPointerInfo(),
LD->getAlign());
17770 ST->getPointerInfo(),
ST->getAlign());
17772 AddToWorklist(NewLD.
getNode());
17773 AddToWorklist(NewST.
getNode());
17774 WorklistRemover DeadNodes(*
this);
17796bool DAGCombiner::isMulAddWithConstProfitable(
SDNode *MulNode,
SDValue AddNode,
17808 if (
Use == MulNode)
17816 if (
Use->getOperand(0) == ConstNode)
17817 OtherOp =
Use->getOperand(1).getNode();
17819 OtherOp =
Use->getOperand(0).getNode();
17831 if (OtherOp == MulVar)
17859 unsigned NumStores) {
17862 SDLoc StoreDL(StoreNodes[0].MemNode);
17864 for (
unsigned i = 0; i < NumStores; ++i) {
17865 Visited.
insert(StoreNodes[i].MemNode);
17869 for (
unsigned i = 0; i < NumStores; ++i) {
17870 if (Visited.
insert(StoreNodes[i].MemNode->getChain().getNode()).second)
17871 Chains.
push_back(StoreNodes[i].MemNode->getChain());
17874 assert(Chains.
size() > 0 &&
"Chain should have generated a chain");
17878bool DAGCombiner::mergeStoresOfConstantsOrVecElts(
17880 bool IsConstantSrc,
bool UseVector,
bool UseTrunc) {
17885 assert((!UseTrunc || !UseVector) &&
17886 "This optimization cannot emit a vector truncating store");
17889 SDLoc DL(StoreNodes[0].MemNode);
17892 unsigned SizeInBits = NumStores * ElementSizeBits;
17897 for (
unsigned I = 0;
I != NumStores; ++
I) {
17913 unsigned Elts = NumStores * NumMemElts;
17921 if (IsConstantSrc) {
17923 for (
unsigned I = 0;
I != NumStores; ++
I) {
17941 .zextOrTrunc(ElementSizeBits),
17951 DL, StoreTy, BuildVector);
17954 for (
unsigned i = 0; i < NumStores; ++i) {
17989 assert(IsConstantSrc &&
"Merged vector elements should use vector store");
17991 APInt StoreInt(SizeInBits, 0);
17996 for (
unsigned i = 0; i < NumStores; ++i) {
17997 unsigned Idx = IsLE ? (NumStores - 1 - i) : i;
18002 StoreInt <<= ElementSizeBits;
18004 StoreInt |=
C->getAPIntValue()
18005 .zextOrTrunc(ElementSizeBits)
18006 .zextOrTrunc(SizeInBits);
18008 StoreInt |=
C->getValueAPF()
18010 .zextOrTrunc(ElementSizeBits)
18011 .zextOrTrunc(SizeInBits);
18025 SDValue NewChain = getMergeStoreChains(StoreNodes, NumStores);
18032 FirstInChain->
getAlign(), *Flags, AAInfo);
18034 EVT LegalizedStoredValTy =
18036 unsigned LegalizedStoreSize = LegalizedStoredValTy.
getSizeInBits();
18039 DAG.
getConstant(
C->getAPIntValue().zextOrTrunc(LegalizedStoreSize),
DL,
18040 LegalizedStoredValTy);
18042 NewChain,
DL, ExtendedStoreVal, FirstInChain->
getBasePtr(),
18044 FirstInChain->
getAlign(), *Flags, AAInfo);
18048 for (
unsigned i = 0; i < NumStores; ++i)
18049 CombineTo(StoreNodes[i].MemNode, NewStore);
18051 AddToWorklist(NewChain.
getNode());
18055void DAGCombiner::getStoreMergeCandidates(
18062 if (!
BasePtr.getBase().getNode() ||
BasePtr.getBase().isUndef())
18066 StoreSource StoreSrc = getStoreSource(Val);
18067 assert(StoreSrc != StoreSource::Unknown &&
"Expected known source for store");
18073 if (StoreSrc == StoreSource::Load) {
18076 LoadVT = Ld->getMemoryVT();
18078 if (MemVT != LoadVT)
18081 if (!Ld->hasNUsesOfValue(1, 0))
18085 if (!Ld->isSimple() || Ld->isIndexed())
18089 int64_t &
Offset) ->
bool {
18092 if (!
Other->isSimple() ||
Other->isIndexed())
18100 :
Other->getMemoryVT() != MemVT;
18101 switch (StoreSrc) {
18102 case StoreSource::Load: {
18110 if (LoadVT != OtherLd->getMemoryVT())
18113 if (!OtherLd->hasNUsesOfValue(1, 0))
18117 if (!OtherLd->isSimple() || OtherLd->isIndexed())
18122 if (!(LBasePtr.equalBaseIndex(LPtr, DAG)))
18126 case StoreSource::Constant:
18132 case StoreSource::Extract:
18134 if (
Other->isTruncatingStore())
18151 auto OverLimitInDependenceCheck = [&](
SDNode *StoreNode,
18152 SDNode *RootNode) ->
bool {
18153 auto RootCount = StoreRootCountMap.
find(StoreNode);
18154 return RootCount != StoreRootCountMap.
end() &&
18155 RootCount->second.first == RootNode &&
18161 if (UseIter.getOperandNo() != 0)
18166 if (CandidateMatch(OtherStore, Ptr, PtrDiff) &&
18167 !OverLimitInDependenceCheck(OtherStore, RootNode))
18168 StoreNodes.
push_back(MemOpLink(OtherStore, PtrDiff));
18190 unsigned NumNodesExplored = 0;
18191 const unsigned MaxSearchNodes = 1024;
18193 RootNode = Ldn->getChain().getNode();
18195 I !=
E && NumNodesExplored < MaxSearchNodes; ++
I, ++NumNodesExplored) {
18197 for (
auto I2 = (*I)->use_begin(), E2 = (*I)->use_end(); I2 != E2; ++I2)
18198 TryToAddCandidate(I2);
18202 TryToAddCandidate(
I);
18207 I !=
E && NumNodesExplored < MaxSearchNodes; ++
I, ++NumNodesExplored)
18208 TryToAddCandidate(
I);
18215bool DAGCombiner::checkMergeStoreCandidatesForDependencies(
18231 while (!Worklist.
empty()) {
18233 if (!Visited.
insert(
N).second)
18242 unsigned int Max = 1024 + Visited.
size();
18244 for (
unsigned i = 0; i < NumStores; ++i) {
18245 SDNode *
N = StoreNodes[i].MemNode;
18262 for (
unsigned j = 0; j <
N->getNumOperands(); ++j)
18263 Worklist.
push_back(
N->getOperand(j).getNode());
18266 for (
unsigned i = 0; i < NumStores; ++i)
18272 if (Visited.
size() >= Max) {
18273 auto &RootCount = StoreRootCountMap[StoreNodes[i].MemNode];
18274 if (RootCount.first == RootNode)
18275 RootCount.second++;
18277 RootCount = {RootNode, 1};
18286 int64_t ElementSizeBytes)
const {
18289 size_t StartIdx = 0;
18290 while ((StartIdx + 1 < StoreNodes.
size()) &&
18291 StoreNodes[StartIdx].OffsetFromBase + ElementSizeBytes !=
18292 StoreNodes[StartIdx + 1].OffsetFromBase)
18296 if (StartIdx + 1 >= StoreNodes.
size())
18305 unsigned NumConsecutiveStores = 1;
18306 int64_t StartAddress = StoreNodes[0].OffsetFromBase;
18309 for (
unsigned i = 1, e = StoreNodes.
size(); i < e; ++i) {
18310 int64_t CurrAddress = StoreNodes[i].OffsetFromBase;
18311 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
18313 NumConsecutiveStores = i + 1;
18315 if (NumConsecutiveStores > 1)
18316 return NumConsecutiveStores;
18324bool DAGCombiner::tryStoreMergeOfConstants(
18326 EVT MemVT,
SDNode *RootNode,
bool AllowVectors) {
18331 bool MadeChange =
false;
18334 while (NumConsecutiveStores >= 2) {
18338 unsigned LastLegalType = 1;
18339 unsigned LastLegalVectorType = 1;
18340 bool LastIntegerTrunc =
false;
18341 bool NonZero =
false;
18342 unsigned FirstZeroAfterNonZero = NumConsecutiveStores;
18343 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
18346 bool IsElementZero =
false;
18348 IsElementZero =
C->isZero();
18350 IsElementZero =
C->getConstantFPValue()->isNullValue();
18351 if (IsElementZero) {
18352 if (NonZero && FirstZeroAfterNonZero == NumConsecutiveStores)
18353 FirstZeroAfterNonZero = i;
18355 NonZero |= !IsElementZero;
18358 unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
18360 bool IsFast =
false;
18372 LastIntegerTrunc =
false;
18373 LastLegalType = i + 1;
18377 EVT LegalizedStoredValTy =
18385 LastIntegerTrunc =
true;
18386 LastLegalType = i + 1;
18397 unsigned Elts = (i + 1) * NumMemElts;
18404 LastLegalVectorType = i + 1;
18408 bool UseVector = (LastLegalVectorType > LastLegalType) && AllowVectors;
18409 unsigned NumElem = (UseVector) ? LastLegalVectorType : LastLegalType;
18410 bool UseTrunc = LastIntegerTrunc && !UseVector;
18422 unsigned NumSkip = 1;
18423 while ((NumSkip < NumConsecutiveStores) &&
18424 (NumSkip < FirstZeroAfterNonZero) &&
18425 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign))
18429 NumConsecutiveStores -= NumSkip;
18434 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
18437 NumConsecutiveStores -= NumElem;
18441 MadeChange |= mergeStoresOfConstantsOrVecElts(StoreNodes, MemVT, NumElem,
18443 UseVector, UseTrunc);
18447 NumConsecutiveStores -= NumElem;
18452bool DAGCombiner::tryStoreMergeOfExtracts(
18458 bool MadeChange =
false;
18461 while (NumConsecutiveStores >= 2) {
18465 unsigned NumStoresToMerge = 1;
18466 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
18468 unsigned Elts = (i + 1) * NumMemElts;
18470 bool IsFast =
false;
18473 if (Ty.getSizeInBits() > MaximumLegalStoreInBits)
18481 NumStoresToMerge = i + 1;
18486 if (NumStoresToMerge < 2) {
18493 unsigned NumSkip = 1;
18494 while ((NumSkip < NumConsecutiveStores) &&
18495 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign))
18499 NumConsecutiveStores -= NumSkip;
18504 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumStoresToMerge,
18507 StoreNodes.
begin() + NumStoresToMerge);
18508 NumConsecutiveStores -= NumStoresToMerge;
18512 MadeChange |= mergeStoresOfConstantsOrVecElts(
18513 StoreNodes, MemVT, NumStoresToMerge,
false,
18516 StoreNodes.
erase(StoreNodes.
begin(), StoreNodes.
begin() + NumStoresToMerge);
18517 NumConsecutiveStores -= NumStoresToMerge;
18523 unsigned NumConsecutiveStores,
EVT MemVT,
18524 SDNode *RootNode,
bool AllowVectors,
18525 bool IsNonTemporalStore,
18526 bool IsNonTemporalLoad) {
18531 bool MadeChange =
false;
18540 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
18547 int64_t LdOffset = 0;
18548 if (LdBasePtr.getBase().getNode()) {
18550 if (!LdBasePtr.equalBaseIndex(LdPtr, DAG, LdOffset))
18558 LoadNodes.
push_back(MemOpLink(Ld, LdOffset));
18561 while (NumConsecutiveStores >= 2 && LoadNodes.
size() >= 2) {
18562 Align RequiredAlignment;
18563 bool NeedRotate =
false;
18564 if (LoadNodes.
size() == 2) {
18568 StoreNodes[0].MemNode->getAlign() >= RequiredAlignment) {
18574 int64_t Offset0 = LoadNodes[0].OffsetFromBase;
18575 int64_t Offset1 = LoadNodes[1].OffsetFromBase;
18577 if (Offset0 - Offset1 == ElementSizeBytes &&
18593 unsigned LastConsecutiveLoad = 1;
18596 unsigned LastLegalVectorType = 1;
18597 unsigned LastLegalIntegerType = 1;
18598 bool isDereferenceable =
true;
18599 bool DoIntegerTruncate =
false;
18600 int64_t StartAddress = LoadNodes[0].OffsetFromBase;
18602 for (
unsigned i = 1; i < LoadNodes.
size(); ++i) {
18604 if (LoadNodes[i].MemNode->getChain() != LoadChain)
18607 int64_t CurrAddress = LoadNodes[i].OffsetFromBase;
18608 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
18610 LastConsecutiveLoad = i;
18612 if (isDereferenceable && !LoadNodes[i].MemNode->isDereferenceable())
18613 isDereferenceable =
false;
18616 unsigned Elts = (i + 1) * NumMemElts;
18623 bool IsFastSt =
false;
18624 bool IsFastLd =
false;
18638 LastLegalVectorType = i + 1;
18642 unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
18653 LastLegalIntegerType = i + 1;
18654 DoIntegerTruncate =
false;
18671 LastLegalIntegerType = i + 1;
18672 DoIntegerTruncate =
true;
18680 LastLegalVectorType > LastLegalIntegerType && AllowVectors;
18681 unsigned LastLegalType =
18682 std::max(LastLegalVectorType, LastLegalIntegerType);
18686 unsigned NumElem = std::min(NumConsecutiveStores, LastConsecutiveLoad + 1);
18687 NumElem = std::min(LastLegalType, NumElem);
18698 unsigned NumSkip = 1;
18699 while ((NumSkip < LoadNodes.
size()) &&
18700 (LoadNodes[NumSkip].MemNode->getAlign() <= FirstLoadAlign) &&
18701 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign))
18705 NumConsecutiveStores -= NumSkip;
18710 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
18714 NumConsecutiveStores -= NumElem;
18723 unsigned Elts = NumElem * NumMemElts;
18726 unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
18730 SDLoc LoadDL(LoadNodes[0].MemNode);
18731 SDLoc StoreDL(StoreNodes[0].MemNode);
18736 SDValue NewStoreChain = getMergeStoreChains(StoreNodes, NumElem);
18737 AddToWorklist(NewStoreChain.
getNode());
18742 if (IsNonTemporalLoad)
18750 if (UseVectorTy || !DoIntegerTruncate) {
18756 unsigned LoadWidth = ElementSizeBytes * 8 * 2;
18758 "Unexpected type for rotate-able load pair");
18765 NewStoreChain, StoreDL, StoreOp, FirstInChain->
getBasePtr(),
18773 FirstLoadAlign, LdMMOFlags);
18775 NewStoreChain, StoreDL, NewLoad, FirstInChain->
getBasePtr(),
18781 for (
unsigned i = 0; i < NumElem; ++i) {
18789 for (
unsigned i = 0; i < NumElem; ++i) {
18790 SDValue Val = StoreNodes[i].MemNode->getOperand(1);
18791 CombineTo(StoreNodes[i].MemNode, NewStore);
18793 recursivelyDeleteUnusedNodes(Val.
getNode());
18799 NumConsecutiveStores -= NumElem;
18804bool DAGCombiner::mergeConsecutiveStores(
StoreSDNode *St) {
18820 if (ElementSizeBytes * 8 != (int64_t)MemVT.
getSizeInBits())
18826 const StoreSource StoreSrc = getStoreSource(StoredVal);
18827 if (StoreSrc == StoreSource::Unknown)
18833 getStoreMergeCandidates(St, StoreNodes, RootNode);
18836 if (StoreNodes.
size() < 2)
18841 llvm::sort(StoreNodes, [](MemOpLink LHS, MemOpLink RHS) {
18842 return LHS.OffsetFromBase < RHS.OffsetFromBase;
18846 Attribute::NoImplicitFloat);
18848 bool IsNonTemporalLoad = StoreSrc == StoreSource::Load &&
18858 bool MadeChange =
false;
18859 while (StoreNodes.
size() > 1) {
18860 unsigned NumConsecutiveStores =
18861 getConsecutiveStores(StoreNodes, ElementSizeBytes);
18863 if (NumConsecutiveStores == 0)
18867 assert(NumConsecutiveStores >= 2 &&
"Expected at least 2 stores");
18868 switch (StoreSrc) {
18869 case StoreSource::Constant:
18870 MadeChange |= tryStoreMergeOfConstants(StoreNodes, NumConsecutiveStores,
18871 MemVT, RootNode, AllowVectors);
18874 case StoreSource::Extract:
18875 MadeChange |= tryStoreMergeOfExtracts(StoreNodes, NumConsecutiveStores,
18879 case StoreSource::Load:
18880 MadeChange |= tryStoreMergeOfLoads(StoreNodes, NumConsecutiveStores,
18881 MemVT, RootNode, AllowVectors,
18882 IsNonTemporalStore, IsNonTemporalLoad);
18897 if (
ST->isTruncatingStore()) {
18899 ST->getBasePtr(),
ST->getMemoryVT(),
18900 ST->getMemOperand());
18902 ReplStore = DAG.
getStore(BetterChain, SL,
ST->getValue(),
ST->getBasePtr(),
18903 ST->getMemOperand());
18911 AddToWorklist(Token.
getNode());
18914 return CombineTo(ST, Token,
false);
18951 bitcastToAPInt().getZExtValue(),
SDLoc(CFP),
18953 return DAG.
getStore(Chain,
DL, Tmp, Ptr,
ST->getMemOperand());
18964 Ptr,
ST->getMemOperand());
18967 if (
ST->isSimple() &&
18982 ST->getOriginalAlign(), MMOFlags, AAInfo);
18985 ST->getPointerInfo().getWithOffset(4),
18986 ST->getOriginalAlign(), MMOFlags, AAInfo);
19003 if (
Value.getOpcode() == ISD::BITCAST && !
ST->isTruncatingStore() &&
19004 ST->isUnindexed()) {
19005 EVT SVT =
Value.getOperand(0).getValueType();
19012 if (((!LegalOperations &&
ST->isSimple()) ||
19015 DAG, *
ST->getMemOperand())) {
19017 ST->getMemOperand());
19022 if (
Value.isUndef() &&
ST->isUnindexed())
19028 if (*Alignment >
ST->getAlign() &&
19029 isAligned(*Alignment,
ST->getSrcValueOffset())) {
19032 ST->getMemoryVT(), *Alignment,
19033 ST->getMemOperand()->getFlags(),
ST->getAAInfo());
19043 if (
SDValue NewST = TransformFPLoadStorePair(
N))
19047 if (
SDValue Store = mergeTruncStores(ST))
19050 if (
ST->isUnindexed()) {
19053 if (findBetterNeighborChains(ST)) {
19058 Chain =
ST->getChain();
19062 if (
ST->isTruncatingStore() &&
ST->isUnindexed() &&
19063 Value.getValueType().isInteger() &&
19070 Value.getOperand(0).getValueType() ==
ST->getMemoryVT() &&
19073 ST->getMemOperand());
19075 APInt TruncDemandedBits =
19077 ST->getMemoryVT().getScalarSizeInBits());
19082 AddToWorklist(
Value.getNode());
19085 ST->getMemOperand());
19105 ST->isUnindexed() &&
ST->isSimple() &&
19109 Chain.reachesChainWithoutSideEffects(
SDValue(Ld, 1))) {
19117 if (
ST->isUnindexed() &&
ST->isSimple() &&
19118 ST1->isUnindexed() && ST1->isSimple()) {
19120 ST1->getValue() ==
Value &&
ST->getMemoryVT() == ST1->getMemoryVT() &&
19121 ST->getAddressSpace() == ST1->getAddressSpace()) {
19128 !ST1->getBasePtr().isUndef() &&
19131 !
ST->getMemoryVT().isScalableVector() &&
19132 !ST1->getMemoryVT().isScalableVector() &&
19133 ST->getAddressSpace() == ST1->getAddressSpace()) {
19136 unsigned STBitSize =
ST->getMemoryVT().getFixedSizeInBits();
19137 unsigned ChainBitSize = ST1->getMemoryVT().getFixedSizeInBits();
19142 if (STBase.contains(DAG, STBitSize, ChainBase, ChainBitSize)) {
19143 CombineTo(ST1, ST1->getChain());
19156 ST->getMemoryVT(), LegalOperations)) {
19158 Ptr,
ST->getMemoryVT(),
ST->getMemOperand());
19169 bool Changed = mergeConsecutiveStores(ST);
19179 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
19188 if (
SDValue NewSt = replaceStoreOfFPConstant(ST))
19195 return ReduceLoadOpStoreWidth(
N);
19200 if (!LifetimeEnd->hasOffset())
19204 LifetimeEnd->getOffset(),
false);
19208 while (!Chains.
empty()) {
19210 if (!Chain.hasOneUse())
19212 switch (Chain.getOpcode()) {
19214 for (
unsigned Nops = Chain.getNumOperands(); Nops;)
19215 Chains.
push_back(Chain.getOperand(--Nops));
19217 case ISD::LIFETIME_START:
19218 case ISD::LIFETIME_END:
19227 if (!
ST->isSimple() ||
ST->isIndexed())
19229 const TypeSize StoreSize =
ST->getMemoryVT().getStoreSize();
19237 if (LifetimeEndBase.contains(DAG, LifetimeEnd->getSize() * 8, StoreBase,
19240 dbgs() <<
"\nwithin LIFETIME_END of : ";
19241 LifetimeEndBase.dump();
dbgs() <<
"\n");
19242 CombineTo(ST,
ST->getChain());
19285 if (!
ST->isSimple())
19318 !
Lo.getOperand(0).getValueType().isScalarInteger() ||
19319 Lo.getOperand(0).getValueSizeInBits() > HalfValBitSize ||
19321 !
Hi.getOperand(0).getValueType().isScalarInteger() ||
19322 Hi.getOperand(0).getValueSizeInBits() > HalfValBitSize)
19328 ?
Lo.getOperand(0).getValueType()
19329 :
Lo.getValueType();
19331 ?
Hi.getOperand(0).getValueType()
19332 :
Hi.getValueType();
19349 ST->getOriginalAlign(), MMOFlags, AAInfo);
19353 St0,
DL,
Hi, Ptr,
ST->getPointerInfo().getWithOffset(HalfValBitSize / 8),
19354 ST->getOriginalAlign(), MMOFlags, AAInfo);
19359SDValue DAGCombiner::combineInsertEltToShuffle(
SDNode *
N,
unsigned InsIndex) {
19361 "Expected extract_vector_elt");
19382 int ElementOffset = -1;
19391 while (!ArgWorkList.
empty()) {
19394 std::tie(ArgOffset, ArgVal) = ArgWorkList.
pop_back_val();
19396 if (ArgVal == InsertVal0) {
19397 ElementOffset = ArgOffset;
19403 int CurrentArgOffset =
19407 CurrentArgOffset -= Step;
19413 assert(CurrentArgOffset == ArgOffset);
19419 if (ElementOffset == -1 &&
Y.isUndef() &&
19421 ElementOffset =
Mask.size();
19425 if (ElementOffset != -1) {
19429 NewMask[InsIndex] = ElementOffset + ExtrIndex->getZExtValue();
19430 assert(NewMask[InsIndex] <
19432 NewMask[InsIndex] >= 0 &&
"NewMask[InsIndex] is out of bound");
19438 return LegalShuffle;
19457 if (NumSrcElts == 1)
19460 unsigned NumMaskVals = ExtendRatio * NumSrcElts;
19468 for (
unsigned i = 0; i != NumMaskVals; ++i) {
19469 if (i / NumSrcElts == InsIndex)
19470 Mask[i] = (i % NumSrcElts) + NumMaskVals;
19485 ConcatOps[0] = SubVec;
19491 AddToWorklist(PaddedSubV.
getNode());
19492 AddToWorklist(DestVecBC.
getNode());
19493 AddToWorklist(Shuf.
getNode());
19536 unsigned Elt = IndexC->getZExtValue();
19538 if (
SDValue Shuf = combineInsertEltToShuffle(
N, Elt))
19542 if (NumElts == 1) {
19560 if (Elt < OtherElt) {
19564 AddToWorklist(NewOp.
getNode());
19583 EVT EltVT = Elt.getValueType();
19584 MaxEltVT = MaxEltVT.
bitsGE(EltVT) ? MaxEltVT : EltVT;
19592 assert(Ops.
size() == NumElts &&
"Unexpected vector size");
19606 for (
SDValue CurVec = InVec; CurVec;) {
19608 if (CurVec.isUndef())
19609 return CanonicalizeBuildVector(Ops);
19613 for (
unsigned I = 0;
I != NumElts; ++
I)
19614 AddBuildVectorOp(Ops, CurVec.getOperand(
I),
I);
19615 return CanonicalizeBuildVector(Ops);
19620 AddBuildVectorOp(Ops, CurVec.getOperand(0), 0);
19621 return CanonicalizeBuildVector(Ops);
19627 if (CurIdx->getAPIntValue().ult(NumElts)) {
19628 unsigned Idx = CurIdx->getZExtValue();
19629 AddBuildVectorOp(Ops, CurVec.getOperand(1), Idx);
19633 return CanonicalizeBuildVector(Ops);
19635 CurVec = CurVec->getOperand(0);
19647SDValue DAGCombiner::scalarizeExtractedVectorLoad(
SDNode *EVE,
EVT InVecVT,
19671 int Elt = ConstEltNo->getZExtValue();
19682 bool IsFast =
false;
19696 if (ResultVT.
bitsGT(VecEltVT)) {
19703 NewPtr, MPI, VecEltVT, Alignment,
19713 if (ResultVT.
bitsLT(VecEltVT))
19725 bool LegalOperations) {
19763 EVT ScalarVT =
N->getValueType(0);
19810 "BUILD_VECTOR used for scalable vectors");
19811 unsigned IndexVal =
19817 if (ScalarVT == InEltVT)
19843 unsigned ExtractIndex = IndexC->getZExtValue();
19845 unsigned BCTruncElt = IsLE ? 0 : NumElts - 1;
19856 "Extract element and scalar to vector can't change element type "
19857 "from FP to integer.");
19858 unsigned XBitWidth =
X.getValueSizeInBits();
19859 BCTruncElt = IsLE ? 0 : XBitWidth / VecEltBitWidth - 1;
19864 if (ExtractIndex == BCTruncElt && XBitWidth > VecEltBitWidth) {
19865 assert(XBitWidth % VecEltBitWidth == 0 &&
19866 "Scalar bitwidth must be a multiple of vector element bitwidth");
19881 int OrigElt = Shuf->getMaskElt(IndexC->getZExtValue());
19889 if (OrigElt < (
int)NumElts) {
19893 OrigElt -= NumElts;
19909 if (!LegalOperations ||
19921 return Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
19922 Use->getOperand(0) == VecOp &&
19923 isa<ConstantSDNode>(Use->getOperand(1));
19928 if (CstElt->getAPIntValue().ult(NumElts))
19929 DemandedElts.
setBit(CstElt->getZExtValue());
19951 bool BCNumEltsChanged =
false;
19957 if (VecOp.
getOpcode() == ISD::BITCAST) {
19966 BCNumEltsChanged =
true;
19972 if (!LegalOperations && !IndexC && VecOp.
hasOneUse() &&
19976 if (VecLoad &&
VecLoad->isSimple())
19977 return scalarizeExtractedVectorLoad(
N, VecVT,
Index, VecLoad);
19982 if (!LegalOperations || !IndexC)
19988 int Elt = IndexC->getZExtValue();
20012 if (BCNumEltsChanged)
20016 int Idx = (Elt > (int)NumElts) ? -1 : Shuf->getMaskElt(Elt);
20021 if (!VecOp.hasOneUse())
20024 VecOp = VecOp.getOperand(0);
20027 LN0 = cast<LoadSDNode>(VecOp);
20028 Elt = (Idx < (int)NumElts) ? Idx : Idx - (int)NumElts;
20029 Index = DAG.getConstant(Elt, DL, Index.getValueType());
20048 Index.getValueType());
20054 return DAG.
getNode(ISD::BITCAST, SL, ScalarVT, Elt);
20066 return scalarizeExtractedVectorLoad(
N, VecVT,
Index, LN0);
20080 unsigned NumInScalars =
N->getNumOperands();
20082 EVT VT =
N->getValueType(0);
20090 bool AllAnyExt =
true;
20092 for (
unsigned i = 0; i != NumInScalars; ++i) {
20095 if (
In.isUndef())
continue;
20101 if (!ZeroExt && !AnyExt) {
20107 EVT InTy =
In.getOperand(0).getValueType();
20113 else if (InTy != SourceType) {
20120 AllAnyExt &= AnyExt;
20127 bool ValidTypes = SourceType !=
MVT::Other &&
20143 assert(ElemRatio > 1 &&
"Invalid element size ratio");
20151 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
20155 Cast.
isUndef()) &&
"Invalid cast opcode");
20161 unsigned Index = isLE ? (i * ElemRatio) :
20162 (i * ElemRatio + (ElemRatio - 1));
20171 "Invalid vector size");
20199 EVT VT =
N->getValueType(0);
20207 unsigned NumInScalars =
N->getNumOperands();
20210 auto PeekThroughBitcast = [](
SDValue Op) {
20211 if (
Op.getOpcode() == ISD::BITCAST)
20212 return Op.getOperand(0);
20218 for (
unsigned i = 0; i != NumInScalars; ++i) {
20219 SDValue In = PeekThroughBitcast(
N->getOperand(i));
20221 if (
In.isUndef())
continue;
20226 In = PeekThroughBitcast(
In.getOperand(0));
20237 SDValue part = PeekThroughBitcast(
In.getOperand(0));
20241 }
else if (Src != part) {
20246 SDValue ShiftAmtVal =
In.getOperand(1);
20250 uint64_t ShiftAmt =
In.getConstantOperandVal(1);
20253 if (ShiftAmt != i * ScalarTypeBitsize)
20259 if (Src.getValueType().getSizeInBits() != VT.
getSizeInBits())
20268 unsigned LeftIdx,
bool DidSplitVec) {
20271 EVT VT =
N->getValueType(0);
20276 unsigned ShuffleNumElems = NumElems;
20286 assert(InVT2Size <= InVT1Size &&
20287 "Inputs must be sorted to be in non-increasing vector size order.");
20291 if (InVT1 != VT || InVT2 != VT) {
20292 if ((VTSize % InVT1Size == 0) && InVT1 == InVT2) {
20295 unsigned NumConcats = VTSize / InVT1Size;
20296 assert(NumConcats >= 2 &&
"Concat needs at least two inputs!");
20298 ConcatOps[0] = VecIn1;
20299 ConcatOps[1] = VecIn2 ? VecIn2 : DAG.
getUNDEF(InVT1);
20302 }
else if (InVT1Size == VTSize * 2) {
20314 Vec2Offset = NumElems;
20316 assert(InVT2Size <= InVT1Size &&
20317 "Second input is not going to be larger than the first one.");
20323 if (LegalOperations &&
20330 if (InVT1 != InVT2) {
20334 DAG.
getUNDEF(InVT1), VecIn2, ZeroIdx);
20336 ShuffleNumElems = NumElems * 2;
20338 }
else if (InVT2Size * 2 == VTSize && InVT1Size == VTSize) {
20340 ConcatOps[0] = VecIn2;
20357 for (
unsigned i = 0; i != NumElems; ++i) {
20358 if (VectorMask[i] <= 0)
20361 unsigned ExtIndex =
N->getOperand(i).getConstantOperandVal(1);
20362 if (VectorMask[i] == (
int)LeftIdx) {
20363 Mask[i] = ExtIndex;
20364 }
else if (VectorMask[i] == (
int)LeftIdx + 1) {
20365 Mask[i] = Vec2Offset + ExtIndex;
20378 if (ShuffleNumElems > NumElems)
20391 for (
int i = 0; i != NumBVOps; ++i) {
20420 SDValue Extract = Zext.getOperand(0);
20421 unsigned DestSize = Zext.getValueSizeInBits();
20423 if (DestSize % SrcSize != 0 ||
20429 int ZextRatio = DestSize / SrcSize;
20430 int NumMaskElts = NumBVOps * ZextRatio;
20432 for (
int i = 0; i != NumMaskElts; ++i) {
20433 if (i / ZextRatio == ZextElt) {
20438 if (i % ZextRatio == 0)
20441 ShufMask[i] = NumMaskElts;
20455 ZeroVec, ShufMask, DAG);
20462template <
typename R,
typename T>
20464 auto I =
find(Range, Val);
20465 if (
I == Range.end())
20466 return static_cast<decltype(std::distance(Range.begin(),
I))
>(-1);
20467 return std::distance(Range.begin(),
I);
20475 EVT VT =
N->getValueType(0);
20488 bool UsesZeroVector =
false;
20489 unsigned NumElems =
N->getNumOperands();
20501 for (
unsigned i = 0; i != NumElems; ++i) {
20511 UsesZeroVector =
true;
20521 SDValue ExtractedFromVec =
Op.getOperand(0);
20526 const APInt &ExtractIdx =
Op.getConstantOperandAPInt(1);
20540 Idx = VecIn.
size();
20544 VectorMask[i] = Idx;
20548 if (VecIn.
size() < 2)
20555 bool DidSplitVec =
false;
20556 if (VecIn.
size() == 2) {
20557 unsigned MaxIndex = 0;
20558 unsigned NearestPow2 = 0;
20563 for (
unsigned i = 0; i < NumElems; i++) {
20564 if (VectorMask[i] <= 0)
20566 unsigned Index =
N->getOperand(i).getConstantOperandVal(1);
20567 IndexVec[i] =
Index;
20568 MaxIndex = std::max(MaxIndex,
Index);
20572 if (InVT.
isSimple() && NearestPow2 > 2 && MaxIndex < NearestPow2 &&
20573 NumElems * 2 < NearestPow2) {
20574 unsigned SplitSize = NearestPow2 / 2;
20587 DidSplitVec =
true;
20589 for (
unsigned i = 0; i < NumElems; i++) {
20590 if (VectorMask[i] <= 0)
20592 VectorMask[i] = (IndexVec[i] < SplitSize) ? 1 : 2;
20610 for (
int &SourceVectorIndex : VectorMask) {
20611 if (SourceVectorIndex <= 0)
20613 unsigned Idx =
getFirstIndexOf(SortedVecIn, VecIn[SourceVectorIndex]);
20614 assert(Idx > 0 && Idx < SortedVecIn.size() &&
20615 VecIn[SourceVectorIndex] == SortedVecIn[Idx] &&
"Remapping failure");
20616 SourceVectorIndex = Idx;
20619 VecIn = std::move(SortedVecIn);
20637 for (
unsigned In = 0, Len = (VecIn.
size() / 2); In < Len; ++In) {
20638 unsigned LeftIdx = 2 *
In + 1;
20639 SDValue VecLeft = VecIn[LeftIdx];
20641 (LeftIdx + 1) < VecIn.
size() ? VecIn[LeftIdx + 1] :
SDValue();
20643 if (
SDValue Shuffle = createBuildVecShuffle(
DL,
N, VectorMask, VecLeft,
20644 VecRight, LeftIdx, DidSplitVec))
20652 if (UsesZeroVector)
20657 if (Shuffles.
size() == 1)
20658 return Shuffles[0];
20661 for (
int &Vec : VectorMask)
20663 Vec = Shuffles.
size() - 1;
20665 Vec = (Vec - 1) / 2;
20679 if (Shuffles.
size() % 2)
20682 for (
unsigned CurSize = Shuffles.
size(); CurSize > 1; CurSize /= 2) {
20684 Shuffles[CurSize] = DAG.
getUNDEF(VT);
20687 for (
unsigned In = 0, Len = CurSize / 2;
In < Len; ++
In) {
20696 if (IsLeftShuffle) {
20703 R.use_empty() && R.getOperand(1).isUndef() &&
20704 R.getOperand(0).getValueType() == R.getValueType();
20705 if (IsRightShuffle) {
20707 R = R.getOperand(0);
20709 for (
unsigned I = 0;
I != NumElems; ++
I) {
20710 if (VectorMask[
I] ==
Left) {
20714 VectorMask[
I] =
In;
20715 }
else if (VectorMask[
I] ==
Right) {
20717 if (IsRightShuffle)
20718 Mask[
I] = RMask[
I] + NumElems;
20719 VectorMask[
I] =
In;
20726 return Shuffles[0];
20734 if (LegalOperations)
20737 EVT VT =
N->getValueType(0);
20739 bool FoundZeroExtend =
false;
20741 auto checkElem = [&](
SDValue Op) -> int64_t {
20742 unsigned Opc =
Op.getOpcode();
20748 return C->getZExtValue();
20756 int64_t
Offset = checkElem(Op0);
20760 unsigned NumElems =
N->getNumOperands();
20762 EVT InSVT =
In.getValueType().getScalarType();
20770 for (
unsigned i = 1; i != NumElems; ++i) {
20771 if ((
Offset + i) != checkElem(
N->getOperand(i)))
20783 EVT VT =
N->getValueType(0);
20797 if (!LegalOperations) {
20816 if (!LegalTypes && (
N->getNumOperands() > 1)) {
20822 return CNode->getZExtValue();
20826 int Offset = checkElem(Op0);
20827 for (
unsigned i = 0; i <
N->getNumOperands(); ++i) {
20828 if (
Offset + i != checkElem(
N->getOperand(i))) {
20838 ((
Offset %
N->getValueType(0).getVectorNumElements()) ==
20844 if (
SDValue V = convertBuildVecZextToZext(
N))
20847 if (
SDValue V = reduceBuildVecExtToExtBuildVec(
N))
20850 if (
SDValue V = reduceBuildVecTruncToBitCast(
N))
20853 if (
SDValue V = reduceBuildVecToShuffle(
N))
20860 assert(!V.isUndef() &&
"Splat of undef should have been handled earlier");
20869 EVT OpVT =
N->getOperand(0).getValueType();
20876 EVT VT =
N->getValueType(0);
20883 bool AnyInteger =
false;
20884 bool AnyFP =
false;
20885 for (
const SDValue &Op :
N->ops()) {
20886 if (ISD::BITCAST == Op.getOpcode() &&
20887 !Op.getOperand(0).getValueType().isVector())
20896 EVT LastOpVT = Ops.
back().getValueType();
20913 if (Op.getValueType() == SVT)
20933 EVT VT =
N->getValueType(0);
20938 for (
const SDValue &Op :
N->ops()) {
20943 if (!FirstConcat) {
20944 SubVT = Op.getOperand(0).getValueType();
20950 if (SubVT != Op.getOperand(0).getValueType())
20953 assert(FirstConcat &&
"Concat of all-undefs found");
20956 for (
const SDValue &Op :
N->ops()) {
20957 if (Op.isUndef()) {
20961 ConcatOps.
append(Op->op_begin(), Op->op_end());
20971 EVT VT =
N->getValueType(0);
20972 EVT OpVT =
N->getOperand(0).getValueType();
20988 if (Op.isUndef()) {
20989 Mask.append((
unsigned)NumOpElts, -1);
20997 SDValue ExtVec = Op.getOperand(0);
20998 int ExtIdx = Op.getConstantOperandVal(1);
21007 Mask.append((
unsigned)NumOpElts, -1);
21018 if (0 == (NumExtElts % NumElts))
21019 ExtIdx /= (NumExtElts / NumElts);
21020 else if (0 == (NumElts % NumExtElts))
21021 ExtIdx *= (NumElts / NumExtElts);
21026 if (SV0.
isUndef() || SV0 == ExtVec) {
21028 for (
int i = 0; i != NumOpElts; ++i)
21029 Mask.push_back(i + ExtIdx);
21030 }
else if (SV1.
isUndef() || SV1 == ExtVec) {
21032 for (
int i = 0; i != NumOpElts; ++i)
21033 Mask.push_back(i + ExtIdx + NumElts);
21045 unsigned CastOpcode =
N->getOperand(0).getOpcode();
21046 switch (CastOpcode) {
21062 EVT SrcVT =
N->getOperand(0).getOperand(0).getValueType();
21070 if (Op.getOpcode() != CastOpcode || !Op.hasOneUse() ||
21071 Op.getOperand(0).getValueType() != SrcVT)
21079 EVT VT =
N->getValueType(0);
21084 switch (CastOpcode) {
21104 return DAG.
getNode(CastOpcode,
DL, VT, NewConcat);
21109 if (
N->getNumOperands() == 1)
21113 EVT VT =
N->getValueType(0);
21119 [](
const SDValue &Op) { return Op.isUndef(); })) {
21121 assert(
In.getValueType().isVector() &&
"Must concat vectors");
21126 unsigned NumOps =
N->getNumOperands() *
In.getNumOperands();
21128 Ops.resize(NumOps, DAG.
getUNDEF(Ops[0].getValueType()));
21138 EVT SVT =
Scalar.getValueType().getVectorElementType();
21139 if (SVT ==
Scalar.getOperand(0).getValueType())
21144 if (!
Scalar.getValueType().isVector()) {
21163 if (VNTNumElms < 2)
21179 auto IsBuildVectorOrUndef = [](
const SDValue &
Op) {
21190 bool FoundMinVT =
false;
21193 EVT OpSVT =
Op.getOperand(0).getValueType();
21194 MinVT = (!FoundMinVT || OpSVT.
bitsLE(MinVT)) ? OpSVT : MinVT;
21197 assert(FoundMinVT &&
"Concat vector type mismatch");
21201 EVT OpVT =
Op.getValueType();
21210 Opnds.
append(
Op->op_begin(),
Op->op_begin() + NumElts);
21212 for (
unsigned i = 0; i != NumElts; ++i)
21220 "Concat vector type mismatch");
21249 unsigned PartNumElem =
21250 N->getOperand(0).getValueType().getVectorMinNumElements();
21252 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
21263 if (SingleSource.
getNode()) {
21264 if (
Op.getOperand(0) != SingleSource)
21267 SingleSource =
Op.getOperand(0);
21277 unsigned IdentityIndex = i * PartNumElem;
21278 if (
Op.getConstantOperandAPInt(1) != IdentityIndex)
21283 return SingleSource;
21292 V.getOperand(1).getValueType() == SubVT && V.getOperand(2) ==
Index) {
21293 return V.getOperand(1);
21297 V.getOperand(0).getValueType() == SubVT &&
21300 return V.getOperand(SubIdx);
21307 bool LegalOperations) {
21310 unsigned BinOpcode = BinOp.
getOpcode();
21316 if (VecVT != Bop0.
getValueType() || VecVT != Bop1.getValueType())
21330 if (!Sub0 || !Sub1)
21336 return DAG.
getNode(BinOpcode,
SDLoc(Extract), SubVT, Sub0, Sub1,
21343 bool LegalOperations) {
21352 if (!ExtractIndexC)
21368 if (
C &&
C->getValueAPF().isNegZero())
21381 unsigned ExtractIndex = ExtractIndexC->getZExtValue();
21383 "Extract index is not a multiple of the vector length.");
21388 if (WideWidth % NarrowWidth != 0)
21393 unsigned NarrowingRatio = WideWidth / NarrowWidth;
21395 if (WideNumElts % NarrowingRatio != 0)
21400 WideNumElts / NarrowingRatio);
21426 if (NarrowingRatio != 2)
21441 return V.getOperand(ConcatOpNum);
21447 if (SubVecL || SubVecR) {
21494 assert(
Index % NumElts == 0 &&
"The extract subvector index is not a "
21495 "multiple of the result's element count");
21514 if (
Offset.isScalable()) {
21537 bool LegalOperations) {
21539 "Must only be called on EXTRACT_SUBVECTOR's");
21544 EVT NarrowVT =
N->getValueType(0);
21551 if (!WideShuffleVector)
21555 if (!WideShuffleVector->hasOneUse())
21559 if (LegalOperations &&
21563 uint64_t FirstExtractedEltIdx =
N->getConstantOperandVal(1);
21565 assert((FirstExtractedEltIdx % NumEltsExtracted) == 0 &&
21566 "Extract index is not a multiple of the output vector length.");
21571 NewMask.
reserve(NumEltsExtracted);
21573 DemandedSubvectors;
21576 for (
int M : WideShuffleVector->getMask().slice(FirstExtractedEltIdx,
21577 NumEltsExtracted)) {
21578 assert((M >= -1) && (M < (2 * WideNumElts)) &&
21579 "Out-of-bounds shuffle mask?");
21588 int WideShufOpIdx = M / WideNumElts;
21590 int OpEltIdx = M % WideNumElts;
21592 assert((OpEltIdx + WideShufOpIdx * WideNumElts) == M &&
21593 "Shuffle mask vector decomposition failure.");
21596 int OpSubvecIdx = OpEltIdx / NumEltsExtracted;
21598 int OpEltIdxInSubvec = OpEltIdx % NumEltsExtracted;
21600 assert((OpEltIdxInSubvec + OpSubvecIdx * NumEltsExtracted) == OpEltIdx &&
21601 "Shuffle mask subvector decomposition failure.");
21603 assert((OpEltIdxInSubvec + OpSubvecIdx * NumEltsExtracted +
21604 WideShufOpIdx * WideNumElts) == M &&
21605 "Shuffle mask full decomposition failure.");
21607 SDValue Op = WideShuffleVector->getOperand(WideShufOpIdx);
21609 if (Op.isUndef()) {
21616 if (OpSubvecIdx != 0)
21619 const std::pair<SDValue, int> DemandedSubvector =
21620 std::make_pair(Op, OpSubvecIdx);
21622 if (DemandedSubvectors.insert(DemandedSubvector)) {
21623 if (DemandedSubvectors.size() > 2)
21626 int Index = NumEltsExtracted * OpSubvecIdx;
21634 getFirstIndexOf(DemandedSubvectors.getArrayRef(), DemandedSubvector);
21635 assert((NewOpIdx == 0 || NewOpIdx == 1) &&
"Unexpected operand index.");
21637 int AdjM = OpEltIdxInSubvec + NewOpIdx * NumEltsExtracted;
21640 assert(NewMask.
size() == (
unsigned)NumEltsExtracted &&
"Produced bad mask.");
21641 assert(DemandedSubvectors.size() <= 2 &&
21642 "Should have ended up demanding at most two subvectors.");
21645 if (DemandedSubvectors.empty())
21659 for (
const std::pair<SDValue /*Op*/, int /*SubvectorIndex*/>
21660 &DemandedSubvector : DemandedSubvectors) {
21662 int Index = NumEltsExtracted * DemandedSubvector.second;
21665 DemandedSubvector.first, IndexC));
21668 "Should end up with either one or two ops");
21671 if (NewOps.
size() == 1)
21678 EVT NVT =
N->getValueType(0);
21680 uint64_t ExtIdx =
N->getConstantOperandVal(1);
21694 V.getConstantOperandVal(1)) &&
21709 if (V.getOpcode() == ISD::BITCAST &&
21710 V.getOperand(0).getValueType().isVector() &&
21715 unsigned DestNumElts = V.getValueType().getVectorMinNumElements();
21716 if ((SrcNumElts % DestNumElts) == 0) {
21717 unsigned SrcDestRatio = SrcNumElts / DestNumElts;
21725 V.getOperand(0), NewIndex);
21729 if ((DestNumElts % SrcNumElts) == 0) {
21730 unsigned DestSrcRatio = DestNumElts / SrcNumElts;
21735 if ((ExtIdx % DestSrcRatio) == 0) {
21737 unsigned IndexValScaled = ExtIdx / DestSrcRatio;
21744 V.getOperand(0), NewIndex);
21752 V.getOperand(0), NewIndex);
21762 EVT ConcatSrcVT = V.getOperand(0).getValueType();
21764 "Concat and extract subvector do not change element type");
21765 assert((ExtIdx % ExtNumElts) == 0 &&
21766 "Extract index is not a multiple of the input vector length.");
21769 unsigned ConcatOpIdx = ExtIdx / ConcatSrcNumElts;
21775 return V.getOperand(ConcatOpIdx);
21783 ConcatSrcNumElts % ExtNumElts == 0) {
21785 unsigned NewExtIdx = ExtIdx - ConcatOpIdx * ConcatSrcNumElts;
21786 assert(NewExtIdx + ExtNumElts <= ConcatSrcNumElts &&
21787 "Trying to extract from >1 concat operand?");
21788 assert(NewExtIdx % ExtNumElts == 0 &&
21789 "Extract index is not a multiple of the input vector length.");
21792 V.getOperand(ConcatOpIdx), NewIndexC);
21804 EVT InVT = V.getValueType();
21808 if (ExtractSize % EltSize == 0) {
21809 unsigned NumElems = ExtractSize / EltSize;
21812 NumElems == 1 ? EltVT
21820 if (NumElems == 1) {
21821 SDValue Src = V->getOperand(IdxVal);
21822 if (EltVT != Src.getValueType())
21829 V->ops().slice(IdxVal, NumElems));
21838 EVT SmallVT = V.getOperand(1).getValueType();
21839 if (!NVT.
bitsEq(SmallVT))
21847 uint64_t InsIdx = V.getConstantOperandVal(2);
21853 return DAG.
getBitcast(NVT, V.getOperand(1));
21857 DAG.
getBitcast(
N->getOperand(0).getValueType(), V.getOperand(0)),
21887 unsigned HalfNumElts = NumElts / 2;
21890 for (
unsigned i = 0; i != NumElts; ++i) {
21894 if ((Mask[i] % NumElts) >= HalfNumElts)
21896 int M = Mask[i] < (int)NumElts ? Mask[i] : Mask[i] - (int)HalfNumElts;
21897 if (i < HalfNumElts)
21900 Mask1[i - HalfNumElts] = M;
21923 EVT VT =
N->getValueType(0);
21934 unsigned NumConcats = NumElts / NumElemsPerConcat;
21936 auto IsUndefMaskElt = [](
int i) {
return i == -1; };
21941 if (NumElemsPerConcat * 2 == NumElts && N1.
isUndef() &&
21942 llvm::all_of(Mask.slice(NumElemsPerConcat, NumElemsPerConcat),
21946 Mask.slice(0, NumElemsPerConcat));
21953 for (
unsigned I = 0;
I != NumConcats; ++
I) {
21954 unsigned Begin =
I * NumElemsPerConcat;
21955 ArrayRef<int> SubMask = Mask.slice(Begin, NumElemsPerConcat);
21964 for (
int i = 0; i != (int)NumElemsPerConcat; ++i) {
21965 if (IsUndefMaskElt(SubMask[i]))
21967 if ((SubMask[i] % (
int)NumElemsPerConcat) != i)
21969 int EltOpIdx = SubMask[i] / NumElemsPerConcat;
21970 if (0 <= OpIdx && EltOpIdx != OpIdx)
21974 assert(0 <= OpIdx &&
"Unknown concat_vectors op");
22028 bool IsSplat =
false;
22032 if (
SDValue Splat0 = BV0->getSplatValue())
22033 IsSplat = (Splat0 == BV1->getSplatValue());
22037 for (
int M : SVN->
getMask()) {
22040 int Idx = M < (int)NumElts ? M : M - NumElts;
22041 SDValue &S = (M < (int)NumElts ? N0 : N1);
22058 if (!IsSplat && !DuplicateOps.
insert(Op).second)
22069 SVT = (SVT.
bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
22072 Op = Op.isUndef() ? DAG.
getUNDEF(SVT)
22086 bool LegalOperations) {
22100 auto isAnyExtend = [&Mask, &NumElts](
unsigned Scale) {
22101 for (
unsigned i = 0; i != NumElts; ++i) {
22104 if ((i % Scale) == 0 && Mask[i] == (
int)(i / Scale))
22113 for (
unsigned Scale = 2; Scale < NumElts; Scale *= 2) {
22115 if (NumElts % Scale != 0)
22117 if (!isAnyExtend(Scale))
22125 if (!LegalOperations ||
22129 SDLoc(SVN), OutVT, N0));
22164 if (ExtDstSizeInBits % ExtSrcSizeInBits != 0)
22166 unsigned ExtScale = ExtDstSizeInBits / ExtSrcSizeInBits;
22171 auto isTruncate = [&Mask, &NumElts](
unsigned Scale) {
22172 for (
unsigned i = 0; i != NumElts; ++i) {
22175 if ((i * Scale) < NumElts && Mask[i] == (
int)(i * Scale))
22185 if (EltSizeInBits != ExtSrcSizeInBits)
22190 if (isTruncate(ExtScale))
22213 if (!Splat || !Splat->isSplat())
22218 assert(ShufMask.
size() == SplatMask.
size() &&
"Mask length mismatch");
22236 auto CanSimplifyToExistingSplat = [](
ArrayRef<int> UserMask,
22238 for (
unsigned i = 0, e = UserMask.
size(); i != e; ++i)
22239 if (UserMask[i] != -1 && SplatMask[i] == -1 &&
22240 SplatMask[UserMask[i]] != -1)
22244 if (CanSimplifyToExistingSplat(ShufMask, SplatMask))
22250 for (
int Idx : ShufMask)
22251 NewMask.
push_back(Idx == -1 ? -1 : SplatMask[Idx]);
22263 bool LegalOperations) {
22280 if (VTLanes <= InLanes || VTLanes % InLanes != 0 ||
22281 (LegalOperations &&
22284 int Factor = VTLanes / InLanes;
22312 if (!InnerShuf || !InnerShuf->getOperand(1).isUndef())
22317 unsigned NumElts = OuterMask.
size();
22318 assert(NumElts == InnerMask.
size() &&
"Mask length mismatch");
22320 int SplatIndex = -1;
22321 for (
unsigned i = 0; i != NumElts; ++i) {
22323 int OuterMaskElt = OuterMask[i];
22324 if (OuterMaskElt == -1)
22328 int InnerMaskElt = InnerMask[OuterMaskElt];
22329 if (InnerMaskElt == -1)
22333 if (SplatIndex == -1)
22334 SplatIndex = InnerMaskElt;
22337 if (SplatIndex != InnerMaskElt)
22340 CombinedMask[i] = InnerMaskElt;
22342 assert((
all_of(CombinedMask, [](
int M) {
return M == -1; }) ||
22344 "Expected a splat mask");
22348 assert(VT == InnerShuf->getValueType(0) &&
"Expected matching shuffle types");
22353 InnerShuf->getOperand(1), CombinedMask);
22361 int MaskSize = Mask.size();
22362 int EltFromOp0 = -1;
22367 for (
int i = 0; i != MaskSize; ++i) {
22368 if (Mask[i] >= 0 && Mask[i] < MaskSize) {
22370 if (EltFromOp0 != -1)
22373 }
else if (Mask[i] != i + MaskSize) {
22393 if (ShufOp0Index == -1) {
22397 if (ShufOp0Index == -1)
22401 Mask = CommutedMask;
22409 assert(Mask[ShufOp0Index] >= 0 && Mask[ShufOp0Index] < (
int)Mask.size() &&
22410 "Shuffle mask value must be from operand 0");
22415 if (!InsIndexC || InsIndexC->getSExtValue() != Mask[ShufOp0Index])
22445 for (
int i = 0, e = (
int)Mask.size(); i != e; ++i) {
22449 assert(Mask[i] >= 0 && Mask[i] < e &&
"Unexpected shuffle mask value");
22453 if (Mask0[Mask[i]] != Mask0[i])
22462 EVT VT =
N->getValueType(0);
22489 for (
unsigned i = 0; i != NumElts; ++i) {
22491 if (Idx >= (
int)NumElts) {
22541 if (Idx->getAPIntValue() == SplatIndex)
22549 if (V->getOpcode() == ISD::BITCAST) {
22557 assert(V->getNumOperands() == NumElts &&
22558 "BUILD_VECTOR has wrong number of operands");
22560 bool AllSame =
true;
22561 for (
unsigned i = 0; i != NumElts; ++i) {
22562 if (!V->getOperand(i).isUndef()) {
22563 Base = V->getOperand(i);
22568 if (!
Base.getNode())
22570 for (
unsigned i = 0; i != NumElts; ++i) {
22571 if (V->getOperand(i) !=
Base) {
22587 if (V->getValueType(0) != VT)
22625 int HalfNumElts = (int)NumElts / 2;
22627 for (
unsigned i = 0; i != NumElts; ++i) {
22629 if (Idx >= HalfNumElts) {
22630 assert(Idx < (
int)NumElts &&
"Shuffle mask chooses undef op");
22631 Idx -= HalfNumElts;
22652 EVT SubVT = RHS.getOperand(0).getValueType();
22653 int NumSubVecs = RHS.getNumOperands();
22655 assert((NumElts % NumSubElts) == 0 &&
"Subvector mismatch");
22660 if (
all_of(Mask, [NumElts](
int M) {
return M < (int)NumElts; }))
22666 for (
int SubVec = 0; SubVec != NumSubVecs; ++SubVec) {
22667 for (
int SubIdx = 0; SubIdx != (int)NumElts; SubIdx += NumSubElts) {
22669 std::iota(InsertionMask.begin(), InsertionMask.end(), 0);
22672 std::iota(InsertionMask.begin() + SubIdx,
22673 InsertionMask.begin() + SubIdx + NumSubElts,
22674 NumElts + (SubVec * NumSubElts));
22677 bool MatchingShuffle =
true;
22678 for (
int i = 0; i != (int)NumElts; ++i) {
22679 int ExpectIdx = InsertionMask[i];
22680 int ActualIdx =
Mask[i];
22681 if (0 <= ActualIdx && ExpectIdx != ActualIdx) {
22682 MatchingShuffle =
false;
22687 if (MatchingShuffle)
22689 RHS.getOperand(SubVec),
22697 if (
SDValue InsertN1 = ShuffleToInsert(N0, N1, Mask))
22702 if (
SDValue InsertN0 = ShuffleToInsert(N1, N0, CommuteMask))
22710 bool IsInLaneMask =
true;
22715 for (
int I = 0;
I != (int)NumElts; ++
I) {
22719 ClearMask[
I] = M ==
I ?
I : (
I + NumElts);
22720 IsInLaneMask &= (M ==
I) || (M == (
int)(
I + NumElts));
22722 APInt &Demanded = M < (int)NumElts ? DemandedLHS : DemandedRHS;
22723 Demanded.
setBit(M % NumElts);
22727 if (!IsInLaneMask &&
22745 for (
int I = 0;
I != (int)NumElts; ++
I)
22747 AndMask[
I] =
Mask[
I] ==
I ? AllOnesElt : ZeroElt;
22784 EVT ScaleVT = SVT.
bitsLT(InnerSVT) ? VT : InnerVT;
22802 for (
int M : OuterMask)
22803 NewMask.
push_back(M < 0 ? -1 : InnerMask[M]);
22834 auto MergeInnerShuffle =
22841 if (OtherSVN->isSplat())
22847 for (
unsigned i = 0; i != NumElts; ++i) {
22848 int Idx = SVN->getMaskElt(i);
22851 Mask.push_back(Idx);
22856 Idx = (Idx < (int)NumElts) ? (Idx + NumElts) : (Idx - NumElts);
22859 if (Idx < (
int)NumElts) {
22862 Idx = OtherSVN->getMaskElt(Idx);
22865 Mask.push_back(Idx);
22868 CurrentVec = (Idx < (int)NumElts) ? OtherSVN->
getOperand(0)
22877 Mask.push_back(-1);
22883 Idx = Idx % NumElts;
22884 if (!SV0.getNode() || SV0 == CurrentVec) {
22888 Mask.push_back(Idx);
22891 if (!SV1.getNode() || SV1 == CurrentVec) {
22895 Mask.push_back(Idx + NumElts);
22902 int InnerIdx = CurrentSVN->getMaskElt(Idx);
22903 if (InnerIdx < 0) {
22904 Mask.push_back(-1);
22907 SDValue InnerVec = (InnerIdx < (int)NumElts)
22911 Mask.push_back(-1);
22914 InnerIdx %= NumElts;
22915 if (InnerVec == SV0) {
22916 Mask.push_back(InnerIdx);
22919 if (InnerVec == SV1) {
22920 Mask.push_back(InnerIdx + NumElts);
22939 if (TLI.isShuffleMaskLegal(Mask, VT))
22944 return TLI.isShuffleMaskLegal(Mask, VT);
22956 assert(N1->getOperand(0).getValueType() == VT &&
22957 "Shuffle types don't match");
22961 bool HasSameOp0 = N0 == SV0;
22962 bool IsSV1Undef = SV1.
isUndef();
22963 if (HasSameOp0 || IsSV1Undef || N0 == SV1)
22984 for (
int i = 0; i != 2; ++i) {
22986 N->isOnlyUserOf(
N->getOperand(i).getNode())) {
22990 assert(OtherSV->getOperand(0).getValueType() == VT &&
22991 "Shuffle types don't match");
22995 if (MergeInnerShuffle(i != 0, SVN, OtherSV,
N->getOperand(1 - i), TLI,
23003 SV1 ? SV1 : DAG.
getUNDEF(VT), Mask);
23013 if (TLI.isBinOp(SrcOpcode) &&
N->isOnlyUserOf(N0.
getNode()) &&
23015 (SrcOpcode == N1.getOpcode() &&
N->isOnlyUserOf(N1.getNode())))) {
23033 SDValue InnerN = Commute ? N1 : N0;
23034 SDValue Op0 = LeftOp ? Op00 : Op01;
23035 SDValue Op1 = LeftOp ? Op10 : Op11;
23042 MergeInnerShuffle(Commute, SVN, SVN0, Op1, TLI, SV0, SV1,
23044 (
llvm::any_of(SVN0->getMask(), [](
int M) { return M < 0; }) ||
23050 bool MergedLeft =
false;
23053 if (CanMergeInnerShuffle(LeftSV0, LeftSV1, LeftMask,
true,
false) ||
23054 CanMergeInnerShuffle(LeftSV0, LeftSV1, LeftMask,
true,
true)) {
23057 LeftMask.
assign(SVN->getMask().begin(), SVN->getMask().end());
23058 LeftSV0 = Op00, LeftSV1 = Op10;
23061 bool MergedRight =
false;
23064 if (CanMergeInnerShuffle(RightSV0, RightSV1, RightMask,
false,
false) ||
23065 CanMergeInnerShuffle(RightSV0, RightSV1, RightMask,
false,
true)) {
23066 MergedRight =
true;
23068 RightMask.
assign(SVN->getMask().begin(), SVN->getMask().end());
23069 RightSV0 = Op01, RightSV1 = Op11;
23072 if (MergedLeft || MergedRight) {
23075 VT,
DL, LeftSV0 ? LeftSV0 : DAG.
getUNDEF(VT),
23076 LeftSV1 ? LeftSV1 : DAG.
getUNDEF(VT), LeftMask);
23078 VT,
DL, RightSV0 ? RightSV0 : DAG.
getUNDEF(VT),
23079 RightSV1 ? RightSV1 : DAG.
getUNDEF(VT), RightMask);
23080 return DAG.
getNode(SrcOpcode,
DL, VT, LHS, RHS);
23094 EVT VT =
N->getValueType(0);
23120 TLI.buildLegalVectorShuffle(InVecT,
SDLoc(
N), InVec,
23121 DAG.
getUNDEF(InVecT), NewMask, DAG);
23122 if (LegalShuffle) {
23126 return LegalShuffle;
23131 InVecT.getVectorElementType(),
23134 LegalShuffle, ZeroIdx);
23145 EVT VT =
N->getValueType(0);
23149 uint64_t InsIdx =
N->getConstantOperandVal(2);
23224 if ((N0.
isUndef() || N0SrcSVT == N1SrcSVT) &&
23236 }
else if ((N1SrcSVT.
getSizeInBits() % EltSizeInBits) == 0) {
23259 if (InsIdx < OtherIdx) {
23263 AddToWorklist(NewOp.
getNode());
23277 Ops[InsIdx / Factor] = N1;
23292 if (N0->
getOpcode() == ISD::FP16_TO_FP)
23305 return DAG.
getNode(ISD::FP16_TO_FP,
SDLoc(
N),
N->getValueType(0),
23317 if (N0->
getOpcode() == ISD::BF16_TO_FP)
23326 unsigned Opcode =
N->getOpcode();
23341 if (Opcode == ISD::VECREDUCE_AND || Opcode == ISD::VECREDUCE_OR) {
23342 unsigned NewOpcode =
Opcode == ISD::VECREDUCE_AND
23343 ? ISD::VECREDUCE_UMIN : ISD::VECREDUCE_UMAX;
23344 if (!TLI.isOperationLegalOrCustom(Opcode, VT) &&
23345 TLI.isOperationLegalOrCustom(NewOpcode, VT) &&
23347 return DAG.
getNode(NewOpcode,
SDLoc(
N),
N->getValueType(0), N0);
23356 if ((Opcode == ISD::VECREDUCE_OR &&
23358 (Opcode == ISD::VECREDUCE_AND &&
23360 return DAG.
getNode(Opcode,
SDLoc(
N),
N->getValueType(0), Subvec);
23370 bool AreAllEltsDisabled =
false;
23374 AreAllEltsDisabled |=
23378 if (!AreAllEltsDisabled)
23383 return DAG.
getUNDEF(
N->getValueType(0));
23388 if (MemSD->writeMem())
23389 return MemSD->getChain();
23390 return CombineTo(
N, DAG.
getUNDEF(
N->getValueType(0)), MemSD->getChain());
23407 EVT VT =
N->getValueType(0);
23414 if (LegalOperations)
23420 EVT RVT = RHS.getValueType();
23421 unsigned NumElts = RHS.getNumOperands();
23426 auto BuildClearMask = [&](
int Split) {
23427 int NumSubElts = NumElts * Split;
23431 for (
int i = 0; i != NumSubElts; ++i) {
23432 int EltIdx = i / Split;
23433 int SubIdx = i % Split;
23434 SDValue Elt = RHS.getOperand(EltIdx);
23452 Bits =
Bits.extractBits(NumSubBits, (Split - SubIdx - 1) * NumSubBits);
23454 Bits =
Bits.extractBits(NumSubBits, SubIdx * NumSubBits);
23456 if (
Bits.isAllOnes())
23458 else if (Bits == 0)
23467 if (!TLI.isVectorClearMaskLegal(Indices, ClearVT))
23481 for (
int Split = 1; Split <= MaxSplit; ++Split)
23483 if (
SDValue S = BuildClearMask(Split))
23495 unsigned Opcode =
N->getOpcode();
23496 EVT VT =
N->getValueType(0);
23503 int Index0, Index1;
23510 if (!Src0 || !Src1 || Index0 != Index1 ||
23530 Ops[Index0] = ScalarBO;
23543 EVT VT =
N->getValueType(0);
23544 assert(VT.
isVector() &&
"SimplifyVBinOp only works on vectors!");
23548 unsigned Opcode =
N->getOpcode();
23563 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
23564 LHS.getOperand(1).isUndef() && RHS.getOperand(1).isUndef() &&
23565 (LHS.hasOneUse() || RHS.hasOneUse() || LHS == RHS)) {
23567 RHS.getOperand(0), Flags);
23579 Shuf0->hasOneUse() && Shuf0->getOperand(1).isUndef() &&
23588 Shuf1->hasOneUse() && Shuf1->getOperand(1).isUndef() &&
23604 LHS.getOperand(2) == RHS.getOperand(2) &&
23605 (LHS.hasOneUse() || RHS.hasOneUse())) {
23609 EVT NarrowVT =
X.getValueType();
23610 if (NarrowVT ==
Y.getValueType() &&
23611 TLI.isOperationLegalOrCustomOrPromote(Opcode, NarrowVT,
23612 LegalOperations)) {
23625 return Op.isUndef() ||
23626 ISD::isBuildVectorOfConstantSDNodes(Op.getNode());
23635 if (ConcatWithConstantOrUndef(LHS) && ConcatWithConstantOrUndef(RHS) &&
23636 (LHS.hasOneUse() || RHS.hasOneUse())) {
23637 EVT NarrowVT = LHS.getOperand(0).getValueType();
23638 if (NarrowVT == RHS.getOperand(0).getValueType() &&
23639 TLI.isOperationLegalOrCustomOrPromote(Opcode, NarrowVT)) {
23640 unsigned NumOperands = LHS.getNumOperands();
23642 for (
unsigned i = 0; i != NumOperands; ++i) {
23645 RHS.getOperand(i)));
23668 if (
SCC.getNode()) {
23675 SCC.getOperand(0),
SCC.getOperand(1),
23676 SCC.getOperand(4), Flags);
23677 AddToWorklist(
SETCC.getNode());
23679 SCC.getOperand(2),
SCC.getOperand(3));
23694bool DAGCombiner::SimplifySelectOps(
SDNode *TheSelect,
SDValue LHS,
23699 if (NaN->isNaN() && RHS.getOpcode() == ISD::FSQRT) {
23715 CmpLHS =
Cmp.getOperand(0);
23719 if (Zero && Zero->
isZero() &&
23723 CombineTo(TheSelect, Sqrt);
23733 if (LHS.getOpcode() != RHS.getOpcode() ||
23734 !LHS.hasOneUse() || !RHS.hasOneUse())
23741 if (LHS.getOpcode() == ISD::LOAD) {
23746 if (LHS.getOperand(0) != RHS.getOperand(0) ||
23766 LLD->getPointerInfo().getAddrSpace() != 0 ||
23772 !TLI.isOperationLegalOrCustom(TheSelect->
getOpcode(),
23773 LLD->getBasePtr().getValueType()))
23790 Visited.
insert(TheSelect);
23808 if ((
LLD->hasAnyUseOfValue(1) &&
23815 LLD->getBasePtr().getValueType(),
23830 if ((
LLD->hasAnyUseOfValue(1) &&
23837 LLD->getBasePtr().getValueType(),
23863 :
LLD->getExtensionType(),
23869 CombineTo(TheSelect, Load);
23873 CombineTo(LHS.getNode(),
Load.getValue(0),
Load.getValue(1));
23874 CombineTo(RHS.getNode(),
Load.getValue(0),
Load.getValue(1));
23898 if (CC ==
ISD::SETGT && TLI.hasAndNot(N2)) {
23916 if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue() - 1)) == 0)) {
23917 unsigned ShCt = XType.
getSizeInBits() - N2C->getAPIntValue().logBase2() - 1;
23918 if (!TLI.shouldAvoidTransformToShift(XType, ShCt)) {
23921 AddToWorklist(Shift.
getNode());
23923 if (XType.
bitsGT(AType)) {
23925 AddToWorklist(Shift.
getNode());
23929 Shift = DAG.
getNOT(
DL, Shift, AType);
23936 if (TLI.shouldAvoidTransformToShift(XType, ShCt))
23941 AddToWorklist(Shift.
getNode());
23943 if (XType.
bitsGT(AType)) {
23945 AddToWorklist(Shift.
getNode());
23949 Shift = DAG.
getNOT(
DL, Shift, AType);
23959 EVT VT =
N->getValueType(0);
23963 if (!TLI.isBinOp(BinOpc) || (N2.
getOpcode() != BinOpc))
24009 EVT VT =
N->getValueType(0);
24010 bool IsFabs =
N->getOpcode() == ISD::FABS;
24011 bool IsFree = IsFabs ? TLI.isFAbsFree(VT) : TLI.isFNegFree(VT);
24017 EVT IntVT =
Int.getValueType();
24031 SignMask = ~SignMask;
24037 SignMask = ~SignMask;
24042 AddToWorklist(
Int.getNode());
24050SDValue DAGCombiner::convertSelectOfFPConstantsToLoadOffset(
24053 if (!TLI.reduceSelectOfFPConstantLoads(N0.
getValueType()))
24061 if (!TV || !FV || !TLI.isTypeLegal(VT))
24066 TLI.isFPImmLegal(TV->getValueAPF(), TV->getValueType(0), ForCodeSize) ||
24067 TLI.isFPImmLegal(FV->getValueAPF(), FV->getValueType(0), ForCodeSize))
24072 if (!TV->hasOneUse() && !FV->hasOneUse())
24076 const_cast<ConstantFP*
>(TV->getConstantFPValue()) };
24093 AddToWorklist(
Cond.getNode());
24095 AddToWorklist(CstOffset.
getNode());
24097 AddToWorklist(CPIdx.
getNode());
24107 bool NotExtCompare) {
24109 if (N2 == N3)
return N2;
24120 AddToWorklist(
SCC.getNode());
24124 return !(SCCC->isZero()) ? N2 : N3;
24129 convertSelectOfFPConstantsToLoadOffset(
DL, N0, N1, N2, N3, CC))
24132 if (
SDValue V = foldSelectCCToShiftAnd(
DL, N0, N1, N2, N3, CC))
24145 if (ConstAndRHS && ConstAndRHS->getAPIntValue().countPopulation() == 1) {
24147 const APInt &AndMask = ConstAndRHS->getAPIntValue();
24149 if (!TLI.shouldAvoidTransformToShift(VT, ShCt)) {
24168 bool Fold = N2C &&
isNullConstant(N3) && N2C->getAPIntValue().isPowerOf2();
24169 bool Swap = N3C &&
isNullConstant(N2) && N3C->getAPIntValue().isPowerOf2();
24171 if ((Fold || Swap) &&
24172 TLI.getBooleanContents(CmpOpVT) ==
24174 (!LegalOperations || TLI.isOperationLegal(
ISD::SETCC, CmpOpVT))) {
24183 if (NotExtCompare && N2C->isOne())
24199 AddToWorklist(
SCC.getNode());
24200 AddToWorklist(Temp.
getNode());
24205 unsigned ShCt = N2C->getAPIntValue().logBase2();
24206 if (TLI.shouldAvoidTransformToShift(VT, ShCt))
24236 N0 ==
Count.getOperand(0) &&
24237 (!LegalOperations || TLI.isOperationLegal(
ISD::CTTZ, VT)))
24243 N0 ==
Count.getOperand(0) &&
24244 (!LegalOperations || TLI.isOperationLegal(
ISD::CTLZ, VT)))
24252 if (!NotExtCompare && N1C && N2C && N3C &&
24253 N2C->getAPIntValue() == ~N3C->getAPIntValue() &&
24275 bool foldBooleans) {
24277 DagCombineInfo(DAG, Level,
false,
this);
24278 return TLI.SimplifySetCC(VT, N0, N1,
Cond, foldBooleans, DagCombineInfo,
DL);
24292 if (
SDValue S = TLI.BuildSDIV(
N, DAG, LegalOperations, Built)) {
24313 if (
SDValue S = TLI.BuildSDIVPow2(
N,
C->getAPIntValue(), DAG, Built)) {
24333 if (
SDValue S = TLI.BuildUDIV(
N, DAG, LegalOperations, Built)) {
24354 if (
SDValue S = TLI.BuildSREMPow2(
N,
C->getAPIntValue(), DAG, Built)) {
24366 EVT VT = V.getValueType();
24387 EVT VT =
Op.getValueType();
24394 int Enabled = TLI.getRecipEstimateDivEnabled(VT, MF);
24395 if (
Enabled == TLI.ReciprocalEstimate::Disabled)
24400 int Iterations = TLI.getDivRefinementSteps(VT, MF);
24401 if (
SDValue Est = TLI.getRecipEstimate(Op, DAG,
Enabled, Iterations)) {
24402 AddToWorklist(Est.getNode());
24410 for (
int i = 0; i < Iterations; ++i) {
24413 if (i == Iterations - 1) {
24415 AddToWorklist(MulEst.
getNode());
24419 AddToWorklist(NewEst.
getNode());
24422 (i == Iterations - 1 ?
N : FPOne), NewEst, Flags);
24423 AddToWorklist(NewEst.
getNode());
24426 AddToWorklist(NewEst.
getNode());
24429 AddToWorklist(Est.getNode());
24434 AddToWorklist(Est.getNode());
24450 unsigned Iterations,
24452 EVT VT =
Arg.getValueType();
24462 for (
unsigned i = 0; i < Iterations; ++i) {
24482 unsigned Iterations,
24484 EVT VT =
Arg.getValueType();
24495 for (
unsigned i = 0; i < Iterations; ++i) {
24504 if (Reciprocal || (i + 1) < Iterations) {
24527 EVT VT =
Op.getValueType();
24534 int Enabled = TLI.getRecipEstimateSqrtEnabled(VT, MF);
24535 if (
Enabled == TLI.ReciprocalEstimate::Disabled)
24540 int Iterations = TLI.getSqrtRefinementSteps(VT, MF);
24542 bool UseOneConstNR =
false;
24544 TLI.getSqrtEstimate(Op, DAG,
Enabled, Iterations, UseOneConstNR,
24546 AddToWorklist(Est.
getNode());
24549 Est = UseOneConstNR
24550 ? buildSqrtNROneConst(Op, Est, Iterations, Flags, Reciprocal)
24551 : buildSqrtNRTwoConst(
Op, Est, Iterations, Flags, Reciprocal);
24562 Test, TLI.getSqrtResultForDenormInput(Op, DAG), Est);
24571 return buildSqrtEstimateImpl(Op, Flags,
true);
24575 return buildSqrtEstimateImpl(Op, Flags,
false);
24579bool DAGCombiner::mayAlias(
SDNode *Op0,
SDNode *Op1)
const {
24581 struct MemUseCharacteristics {
24590 auto getCharacteristics = [](
SDNode *
N) -> MemUseCharacteristics {
24595 ?
C->getSExtValue()
24597 ? -1 *
C->getSExtValue()
24601 return {LSN->isVolatile(), LSN->isAtomic(), LSN->getBasePtr(),
24604 LSN->getMemOperand()};
24607 return {
false ,
false, LN->getOperand(1),
24608 (LN->hasOffset()) ? LN->getOffset() : 0,
24613 return {
false ,
false,
SDValue(),
24618 MemUseCharacteristics MUC0 = getCharacteristics(Op0),
24619 MUC1 = getCharacteristics(Op1);
24622 if (MUC0.BasePtr.getNode() && MUC0.BasePtr == MUC1.BasePtr &&
24623 MUC0.Offset == MUC1.Offset)
24627 if (MUC0.IsVolatile && MUC1.IsVolatile)
24632 if (MUC0.IsAtomic && MUC1.IsAtomic)
24635 if (MUC0.MMO && MUC1.MMO) {
24636 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
24637 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
24650 if (!MUC0.MMO || !MUC1.MMO)
24656 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
24657 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
24665 int64_t SrcValOffset0 = MUC0.MMO->getOffset();
24666 int64_t SrcValOffset1 = MUC1.MMO->getOffset();
24667 Align OrigAlignment0 = MUC0.MMO->getBaseAlign();
24668 Align OrigAlignment1 = MUC1.MMO->getBaseAlign();
24669 auto &Size0 = MUC0.NumBytes;
24670 auto &Size1 = MUC1.NumBytes;
24671 if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
24672 Size0.has_value() && Size1.has_value() && *Size0 == *Size1 &&
24673 OrigAlignment0 > *Size0 && SrcValOffset0 % *Size0 == 0 &&
24674 SrcValOffset1 % *Size1 == 0) {
24675 int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.
value();
24676 int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.
value();
24680 if ((OffAlign0 + *Size0) <= OffAlign1 || (OffAlign1 + *Size1) <= OffAlign0)
24693 if (
UseAA && AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() && Size0 &&
24696 int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
24697 int64_t Overlap0 = *Size0 + SrcValOffset0 - MinOffset;
24698 int64_t Overlap1 = *Size1 + SrcValOffset1 - MinOffset;
24713void DAGCombiner::GatherAllAliases(
SDNode *
N,
SDValue OriginalChain,
24724 unsigned Depth = 0;
24727 auto ImproveChain = [&](
SDValue &
C) ->
bool {
24728 switch (
C.getOpcode()) {
24739 if ((IsLoad && IsOpLoad) || !
mayAlias(
N,
C.getNode())) {
24741 C =
C.getOperand(0);
24750 C =
C.getOperand(0);
24753 case ISD::LIFETIME_START:
24754 case ISD::LIFETIME_END: {
24759 C =
C.getOperand(0);
24772 while (!Chains.
empty()) {
24776 if (!Visited.
insert(Chain.getNode()).second)
24785 if (
Depth > TLI.getGatherAllAliasesMaxDepth()) {
24796 if (Chain.getNumOperands() > 16) {
24800 for (
unsigned n = Chain.getNumOperands(); n;)
24801 Chains.
push_back(Chain.getOperand(--n));
24806 if (ImproveChain(Chain)) {
24808 if (Chain.getNode())
24828 GatherAllAliases(
N, OldChain, Aliases);
24831 if (Aliases.
size() == 0)
24835 if (Aliases.
size() == 1)
24844struct UnitT { } Unit;
24845bool operator==(
const UnitT &,
const UnitT &) {
return true; }
24846bool operator!=(
const UnitT &,
const UnitT &) {
return false; }
24862bool DAGCombiner::parallelizeChainedStores(
StoreSDNode *St) {
24879 if (!
BasePtr.getBase().getNode())
24883 if (
BasePtr.getBase().isUndef())
24900 if (Chain->getMemoryVT().isScalableVector())
24907 if (!Chain->isSimple() || Chain->isIndexed())
24916 int64_t Length = (Chain->getMemoryVT().getSizeInBits() + 7) / 8;
24919 auto I = Intervals.find(
Offset);
24921 if (
I != Intervals.end() &&
I.start() < (
Offset + Length))
24924 if (
I != Intervals.begin() && (--
I).stop() <=
Offset)
24933 if (ChainedStores.
size() == 0)
24940 for (
unsigned I = ChainedStores.
size();
I;) {
24942 SDValue BetterChain = FindBetterChain(S, NewChain);
24946 ChainedStores[
I] = S;
24950 SDValue BetterChain = FindBetterChain(St, NewChain);
24965 auto hasImprovedChain = [&](
SDValue ST) ->
bool {
24966 return ST->getOperand(0) != NewChain;
24968 bool AddNewChain =
llvm::all_of(TFOps, hasImprovedChain);
24978 AddToWorklist(
Op.getNode());
24979 AddToWorklist(STChain);
24983bool DAGCombiner::findBetterNeighborChains(
StoreSDNode *St) {
24990 if (!
BasePtr.getBase().getNode())
24994 if (
BasePtr.getBase().isUndef())
24998 if (parallelizeChainedStores(St))
25003 if (St->
getChain() != BetterChain) {
25004 replaceStoreChain(St, BetterChain);
25014 DAGCombiner(*
this, AA, OptLevel).Run(Level);
static bool mayAlias(MachineInstr &MIa, SmallVectorImpl< MachineInstr * > &MemInsns, AliasAnalysis *AA)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static cl::opt< bool > UseAA("aarch64-use-aa", cl::init(true), cl::desc("Enable the use of AA during codegen."))
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static uint64_t * getMemory(unsigned numWords)
A utility function for allocating memory and checking for allocation failure.
This file implements a class to represent arbitrary precision integral constant values and operations...
This file contains the simple types necessary to represent the attributes associated with functions a...
SmallVector< MachineOperand, 4 > Cond
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static bool splitMergedValStore(StoreInst &SI, const DataLayout &DL, const TargetLowering &TLI)
For the instruction sequence of store below, F and I values are bundled together as an i64 value befo...
static unsigned bigEndianByteAt(const unsigned ByteWidth, const unsigned I)
static Optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
static unsigned littleEndianByteAt(const unsigned ByteWidth, const unsigned I)
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques=false)
static cl::opt< bool > EnableShrinkLoadReplaceStoreWithStore("combiner-shrink-load-replace-store-with-store", cl::Hidden, cl::init(true), cl::desc("DAG combiner enable load/<replace bytes>/store with " "a narrower store"))
static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0, unsigned ExtOpc, SmallVectorImpl< SDNode * > &ExtendNodes, const TargetLowering &TLI)
static bool CanCombineFCOPYSIGN_EXTEND_ROUND(SDNode *N)
copysign(x, fp_extend(y)) -> copysign(x, y) copysign(x, fp_round(y)) -> copysign(x,...
static cl::opt< unsigned > TokenFactorInlineLimit("combiner-tokenfactor-inline-limit", cl::Hidden, cl::init(2048), cl::desc("Limit the number of operands to inline for Token Factors"))
static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG)
static SDNode * getBuildPairElt(SDNode *N, unsigned i)
static SDValue scalarizeBinOpOfSplats(SDNode *N, SelectionDAG &DAG, const SDLoc &DL)
If a vector binop is performed on splat values, it may be profitable to extract, scalarize,...
static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift, SDValue ExtractFrom, SDValue &Mask, const SDLoc &DL)
Helper function for visitOR to extract the needed side of a rotate idiom from a shl/srl/mul/udiv.
static bool getCombineLoadStoreParts(SDNode *N, unsigned Inc, unsigned Dec, bool &IsLoad, bool &IsMasked, SDValue &Ptr, const TargetLowering &TLI)
static SDValue tryToFoldExtOfMaskedLoad(SelectionDAG &DAG, const TargetLowering &TLI, EVT VT, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc)
static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned, const TargetLowering &TLI)
Return true if divmod libcall is available.
bool refineUniformBase(SDValue &BasePtr, SDValue &Index, bool IndexIsScaled, SelectionDAG &DAG)
static SDValue reduceBuildVecToShuffleWithZero(SDNode *BV, SelectionDAG &DAG)
static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1, SelectionDAG &DAG, const SDLoc &DL)
Given the operands of an add/sub operation, see if the 2nd operand is a masked 0/1 whose source opera...
static SDValue simplifyShuffleOfShuffle(ShuffleVectorSDNode *Shuf)
If we have a unary shuffle of a shuffle, see if it can be folded away completely.
static bool canSplitIdx(LoadSDNode *LD)
static SDValue ShrinkLoadReplaceStoreWithStore(const std::pair< unsigned, unsigned > &MaskInfo, SDValue IVal, StoreSDNode *St, DAGCombiner *DC)
Check to see if IVal is something that provides a value as specified by MaskInfo.
static cl::opt< bool > StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden, cl::desc("Bypass the profitability model of load slicing"), cl::init(false))
Hidden option to stress test load slicing, i.e., when this option is enabled, load slicing bypasses m...
static cl::opt< bool > UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true), cl::desc("Enable DAG combiner's use of TBAA"))
static void adjustCostForPairing(SmallVectorImpl< LoadedSlice > &LoadedSlices, LoadedSlice::Cost &GlobalLSCost)
Adjust the GlobalLSCost according to the target paring capabilities and the layout of the slices.
static SDValue narrowInsertExtractVectorBinOp(SDNode *Extract, SelectionDAG &DAG, bool LegalOperations)
static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI, SDValue N0, SDValue N1, SDNode *N)
static const Optional< ByteProvider > calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth, bool Root=false)
Recursively traverses the expression calculating the origin of the requested byte of the given value.
static bool matchRotateHalf(SelectionDAG &DAG, SDValue Op, SDValue &Shift, SDValue &Mask)
Match "(X shl/srl V1) & V2" where V2 may not be present.
static bool isCompatibleLoad(SDValue N, unsigned ExtOpcode)
Check if N satisfies: N is used once.
static SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG)
static SDValue narrowExtractedVectorBinOp(SDNode *Extract, SelectionDAG &DAG, bool LegalOperations)
If we are extracting a subvector produced by a wide binary operator try to use a narrow binary operat...
static bool areUsedBitsDense(const APInt &UsedBits)
Check that all bits set in UsedBits form a dense region, i.e., UsedBits looks like 0....
static SDValue getInputChainForNode(SDNode *N)
Given a node, return its input chain if it has one, otherwise return a null sd operand.
static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG)
If we are extracting a subvector from a wide vector load, convert to a narrow load to eliminate the e...
static ElementCount numVectorEltsOrZero(EVT T)
static SDValue foldSelectWithIdentityConstant(SDNode *N, SelectionDAG &DAG, bool ShouldCommuteOperands)
This inverts a canonicalization in IR that replaces a variable select arm with an identity constant.
static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG)
Given an extending node with a pop-count operand, if the target does not support a pop-count in the n...
static SDValue foldSelectOfConstantsUsingSra(SDNode *N, SelectionDAG &DAG)
If a (v)select has a condition value that is a sign-bit test, try to smear the condition operand sign...
static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG, SDValue X, SDValue Carry0, SDValue Carry1, SDNode *N)
If we are facing some sort of diamond carry propapagtion pattern try to break it up to generate somet...
static SDValue replaceShuffleOfInsert(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
If a shuffle inserts exactly one element from a source vector operand into another vector operand and...
static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner, const TargetLowering &TLI, EVT VT, bool LegalOperations, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType)
static cl::opt< bool > CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden, cl::desc("Enable DAG combiner's use of IR alias analysis"))
static bool isConstantSplatVectorMaskForType(SDNode *N, EVT ScalarTy)
static SDValue formSplatFromShuffles(ShuffleVectorSDNode *OuterShuf, SelectionDAG &DAG)
Combine shuffle of shuffle of the form: shuf (shuf X, undef, InnerMask), undef, OuterMask --> splat X...
static bool isDivisorPowerOfTwo(SDValue Divisor)
static bool canFoldInAddressingMode(SDNode *N, SDNode *Use, SelectionDAG &DAG, const TargetLowering &TLI)
Return true if 'Use' is a load or a store that uses N as its base pointer and that N may be folded in...
static SDValue foldExtractSubvectorFromShuffleVector(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalOperations)
Given EXTRACT_SUBVECTOR(VECTOR_SHUFFLE(Op0, Op1, Mask)), try to produce VECTOR_SHUFFLE(EXTRACT_SUBVEC...
static SDValue combineConcatVectorOfExtracts(SDNode *N, SelectionDAG &DAG)
static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG, bool LegalOperations)
Transform a vector binary operation into a scalar binary operation by moving the math/logic after an ...
static bool hasNoInfs(const TargetOptions &Options, SDValue N)
static SDValue isSaturatingMinMax(SDValue N0, SDValue N1, SDValue N2, SDValue N3, ISD::CondCode CC, unsigned &BW, bool &Unsigned)
static SDValue combineShuffleOfBitcast(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalOperations)
static SDValue combineShuffleToVectorExtend(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalOperations)
static SDValue PerformUMinFpToSatCombine(SDValue N0, SDValue N1, SDValue N2, SDValue N3, ISD::CondCode CC, SelectionDAG &DAG)
static SDValue tryToFoldExtendSelectLoad(SDNode *N, const TargetLowering &TLI, SelectionDAG &DAG)
Fold (sext (select c, load x, load y)) -> (select c, sextload x, sextload y) (zext (select c,...
static SDValue foldAndToUsubsat(SDNode *N, SelectionDAG &DAG)
For targets that support usubsat, match a bit-hack form of that operation that ends in 'and' and conv...
static SDValue stripTruncAndExt(SDValue Value)
static SDValue foldShuffleOfConcatUndefs(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
Try to convert a wide shuffle of concatenated vectors into 2 narrow shuffles followed by concatenatio...
static SDValue combineShuffleOfSplatVal(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
static auto getFirstIndexOf(R &&Range, const T &Val)
static std::pair< unsigned, unsigned > CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain)
Check to see if V is (and load (ptr), imm), where the load is having specific bytes cleared out.
static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG)
Try to fold a 'not' shifted sign-bit with add/sub with constant operand into a shift and add with a d...
static int getShuffleMaskIndexOfOneElementFromOp0IntoOp1(ArrayRef< int > Mask)
If the shuffle mask is taking exactly one element from the first vector operand and passing through a...
static cl::opt< bool > EnableStoreMerging("combiner-store-merging", cl::Hidden, cl::init(true), cl::desc("DAG combiner enable merging multiple stores " "into a wider store"))
static bool isContractableFMUL(const TargetOptions &Options, SDValue N)
static cl::opt< bool > MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true), cl::desc("DAG combiner may split indexing from loads"))
static bool areSlicesNextToEachOther(const LoadedSlice &First, const LoadedSlice &Second)
Check whether or not First and Second are next to each other in memory.
static bool isBSwapHWordPair(SDValue N, MutableArrayRef< SDNode * > Parts)
static SDValue foldFPToIntToFP(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue getTruncatedUSUBSAT(EVT DstVT, EVT SrcVT, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &DL)
static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG)
static SDValue foldBoolSelectToLogic(SDNode *N, SelectionDAG &DAG)
static SDNode * getPostIndexedLoadStoreOp(SDNode *N, bool &IsLoad, bool &IsMasked, SDValue &Ptr, SDValue &BasePtr, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG, const TargetLowering &TLI)
static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS, SDValue RHS, const TargetLowering &TLI)
static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG, const TargetLowering &TLI, bool Force)
Flips a boolean if it is cheaper to compute.
static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op, KnownBits &Known)
static SDValue getSubVectorSrc(SDValue V, SDValue Index, EVT SubVT)
bool refineIndexType(SDValue &Index, ISD::MemIndexType &IndexType, EVT DataVT, SelectionDAG &DAG)
static SDValue getAsCarry(const TargetLowering &TLI, SDValue V)
static SDValue combineShiftOfShiftedLogic(SDNode *Shift, SelectionDAG &DAG)
If we have a shift-by-constant of a bitwise logic op that itself has a shift-by-constant operand with...
static SDValue foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset=0)
static ConstantSDNode * getAsNonOpaqueConstant(SDValue N)
If N is a ConstantSDNode with isOpaque() == false return it casted to a ConstantSDNode pointer else n...
static SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode CC, const TargetLowering &TLI, SelectionDAG &DAG)
Generate Min/Max node.
static SDValue combineShiftToMULH(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue PerformMinMaxFpToSatCombine(SDValue N0, SDValue N1, SDValue N2, SDValue N3, ISD::CondCode CC, SelectionDAG &DAG)
static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize, SelectionDAG &DAG, bool IsRotate)
static SDValue visitORCommutative(SelectionDAG &DAG, SDValue N0, SDValue N1, SDNode *N)
OR combines for which the commuted variant will be tried as well.
static cl::opt< bool > EnableReduceLoadOpStoreWidth("combiner-reduce-load-op-store-width", cl::Hidden, cl::init(true), cl::desc("DAG combiner enable reducing the width of load/op/store " "sequence"))
static bool shouldCombineToPostInc(SDNode *N, SDValue Ptr, SDNode *PtrUse, SDValue &BasePtr, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI, SelectionDAG &DAG, bool LegalTypes)
Try to fold a sext/zext/aext dag node into a ConstantSDNode or a build_vector of constants.
static SDValue foldExtendedSignBitTest(SDNode *N, SelectionDAG &DAG, bool LegalOperations)
static SDValue combineConcatVectorOfCasts(SDNode *N, SelectionDAG &DAG)
static SDValue combineShiftAnd1ToBitTest(SDNode *And, SelectionDAG &DAG)
Try to replace shift/logic that tests if a bit is clear with mask + setcc.
static SDValue matchBSwapHWordOrAndAnd(const TargetLowering &TLI, SelectionDAG &DAG, SDNode *N, SDValue N0, SDValue N1, EVT VT, EVT ShiftAmountTy)
static SDValue stripConstantMask(SelectionDAG &DAG, SDValue Op, SDValue &Mask)
static SDValue combineShuffleOfScalars(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG)
static SDValue foldVSelectToSignBitSplatMask(SDNode *N, SelectionDAG &DAG)
static SDValue combineConcatVectorOfConcatVectors(SDNode *N, SelectionDAG &DAG)
static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner, const TargetLowering &TLI, EVT VT, bool LegalOperations, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc)
static unsigned getPPCf128HiElementSelector(const SelectionDAG &DAG)
static SDValue combineTruncationShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG)
static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT, SelectionDAG &DAG, bool LegalOperations)
static cl::opt< unsigned > StoreMergeDependenceLimit("combiner-store-merge-dependence-limit", cl::Hidden, cl::init(10), cl::desc("Limit the number of times for the same StoreNode and RootNode " "to bail out in store merging dependence check"))
static SDValue combineABSToABD(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static cl::opt< std::string > CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden, cl::desc("Only use DAG-combiner alias analysis in this" " function"))
static SDValue foldLogicOfShifts(SDNode *N, SDValue LogicOp, SDValue ShiftOp, SelectionDAG &DAG)
Given a bitwise logic operation N with a matching bitwise logic operand, fold a pattern where 2 of th...
static bool isSlicingProfitable(SmallVectorImpl< LoadedSlice > &LoadedSlices, const APInt &UsedBits, bool ForCodeSize)
Check the profitability of all involved LoadedSlice.
static bool isADDLike(SDValue V, const SelectionDAG &DAG)
static bool isBSwapHWordElement(SDValue N, MutableArrayRef< SDNode * > Parts)
Return true if the specified node is an element that makes up a 32-bit packed halfword byteswap.
static SDValue FoldIntToFPToInt(SDNode *N, SelectionDAG &DAG)
static ManagedStatic< DebugCounter > DC
This file defines the DenseMap class.
Optional< std::vector< StOtherPiece > > Other
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
iv Induction Variable Users
static Value * simplifyDivRem(Instruction::BinaryOps Opcode, Value *Op0, Value *Op1, const SimplifyQuery &Q)
Check for common or similar folds of integer division or integer remainder.
This file implements a coalescing interval map for small objects.
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
This file provides None, an enumerator for use in implicit constructors of various (usually templated...
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
This file provides Optional, a template class modeled in the spirit of OCaml's 'opt' variant.
const char LLVMTargetMachineRef LLVMPassBuilderOptionsRef Options
static cl::opt< bool > Aggressive("aggressive-ext-opt", cl::Hidden, cl::desc("Aggressive extension optimization"))
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSimple(Instruction *I)
static cl::opt< bool > UseTBAA("use-tbaa-in-sched-mi", cl::Hidden, cl::init(true), cl::desc("Enable use of TBAA during MI DAG construction"))
This file implements a set that has insertion order iteration characteristics.
This file implements the SmallBitVector class.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file describes how to lower LLVM code to machine code.
static Optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
static bool hasOneUse(unsigned Reg, MachineInstr *Def, MachineRegisterInfo &MRI, MachineDominatorTree &MDT, LiveIntervals &LIS)
static constexpr int Concat[]
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
const fltSemantics & getSemantics() const
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
APInt umul_ov(const APInt &RHS, bool &Overflow) const
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
static void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
bool isNegatedPowerOf2() const
Check if this APInt's negated value is a power of two greater than zero.
APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
bool isMinSignedValue() const
Determine if this is the smallest signed value.
uint64_t getZExtValue() const
Get zero extended value.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
APInt abs() const
Get the absolute value.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
static APInt getNullValue(unsigned numBits)
NOTE: This is soft-deprecated. Please use getZero() instead.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
bool isNegative() const
Determine sign of this APInt.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
int32_t exactLogBase2() const
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
unsigned countLeadingZeros() const
The APInt version of the countLeadingZeros functions in MathExtras.h.
unsigned logBase2() const
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
bool isNullValue() const
NOTE: This is soft-deprecated. Please use isZero() instead.
bool getBoolValue() const
Convert APInt to a boolean value.
APInt smul_ov(const APInt &RHS, bool &Overflow) const
bool isMask(unsigned numBits) const
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
APInt sext(unsigned width) const
Sign extend to a new width.
APInt shl(unsigned shiftAmt) const
Left-shift function.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
unsigned countTrailingOnes() const
Count the number of trailing one bits.
bool isOne() const
Determine if this is a value of 1.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
This class holds the attributes for a function, its return value, and its parameters.
static bool computeAliasing(const SDNode *Op0, const Optional< int64_t > NumBytes0, const SDNode *Op1, const Optional< int64_t > NumBytes1, const SelectionDAG &DAG, bool &IsAlias)
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
A "pseudo-class" with methods for operating on BUILD_VECTORs.
ISD::CondCode get() const
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
const APFloat & getValueAPF() const
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
bool isNegative() const
Return true if the value is negative.
bool isZero() const
Return true if the value is positive or negative zero.
ConstantFP - Floating Point Values [float, double].
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
int64_t getSExtValue() const
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
iterator find(const_arg_type_t< KeyT > Val)
bool erase(const KeyT &Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool isVector() const
One or more elements.
bool isScalar() const
Counting predicates.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
AttributeList getAttributes() const
Return the attribute list for this Function.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
This class is used to form a handle around another node that is persistent and is updated across invo...
This is an important class for using LLVM in a threaded context.
Base class for LoadSDNode and StoreSDNode.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
bool isKnownMultipleOf(ScalarTy RHS) const
This function tells the caller whether the element count is known at compile time to be a multiple of...
LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
static ElementCount getFixed(ScalarTy MinVal)
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
static auto all_valuetypes()
SimpleValueType Iteration.
static MVT getIntegerVT(unsigned BitWidth)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
This class is used to represent an MGATHER node.
const SDValue & getPassThru() const
ISD::LoadExtType getExtensionType() const
const SDValue & getIndex() const
bool isIndexScaled() const
const SDValue & getScale() const
const SDValue & getBasePtr() const
const SDValue & getMask() const
ISD::MemIndexType getIndexType() const
How is Index applied to BasePtr when computing addresses.
This class is used to represent an MLOAD node.
const SDValue & getBasePtr() const
bool isExpandingLoad() const
ISD::LoadExtType getExtensionType() const
const SDValue & getMask() const
const SDValue & getPassThru() const
const SDValue & getOffset() const
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
This class is used to represent an MSCATTER node.
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This class is used to represent an MSTORE node.
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
const SDValue & getOffset() const
const SDValue & getBasePtr() const
const SDValue & getMask() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getBasePtr() const
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
bool isNonTemporal() const
bool isDereferenceable() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
static uint64_t getSizeOrUnknown(const TypeSize &T)
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
size_t use_size() const
Return the number of uses of this node.
void intersectFlagsWith(const SDNodeFlags Flags)
Clear any flags in this node that aren't also set in Flags.
TypeSize getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
SDVTList getVTList() const
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
void setFlags(SDNodeFlags NewFlags)
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
virtual bool disableGenericCombines(CodeGenOpt::Level OptLevel) const
Help to insert SDNodeFlags automatically in transforming.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getSplatSourceVector(SDValue V, int &SplatIndex)
If V is a splatted value, return the source vector and its splat index.
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
bool isKnownNeverZero(SDValue Op) const
Test whether the given SDValue is known to contain non-zero value(s).
const TargetSubtargetInfo & getSubtarget() const
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N) const
Test whether the given value is a constant int or similar node.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=None, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
bool shouldOptForSize() const
OverflowKind computeOverflowKind(SDValue N0, SDValue N1) const
Determine if the result of the addition of 2 node can overflow.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal)
Try to simplify a select/vselect into 1 of its operands or a constant.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDNode * isConstantFPBuildVectorOrConstantFP(SDValue N) const
Test whether the given value is a constant FP or similar node.
SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits)
See if the specified operand can be simplified with the knowledge that only the bits specified by Dem...
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
bool isKnownToBeAPowerOfTwo(SDValue Val) const
Test if the given value is known to have exactly one bit set.
bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly=false, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
bool isConstantValueOfAnyType(SDValue N) const
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
const TargetMachine & getTarget() const
SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
iterator_range< allnodes_iterator > allnodes()
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue is known to never be NaN.
SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
const TargetLibraryInfo & getLibInfo() const
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Return true if 'Op' is known to be zero in DemandedElts.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
void Combine(CombineLevel Level, AAResults *AA, CodeGenOpt::Level OptLevel)
This iterates over the nodes in the SelectionDAG, folding certain types of nodes together,...
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, SDNodeFlags Flags)
Try to simplify a floating-point binary operation into 1 of its operands or a constant.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
bool isUndef(unsigned Opcode, ArrayRef< SDValue > Ops)
Return true if the result of this operation is always undefined.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
DenormalMode getDenormalMode(EVT VT) const
Return the current function's default denormal handling kind for the given floating point type.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue simplifyShift(SDValue X, SDValue Y)
Try to simplify a shift into 1 of its operands or a constant.
void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits=0, unsigned SizeInBits=0, bool InvalidateDbg=true)
Transfer debug values from one node to another, while optionally generating fragment expressions for ...
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
A vector that has set insertion semantics.
bool remove(const value_type &X)
Remove an item from the set vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
LLVM_NODISCARD T pop_back_val()
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
ArrayRef< int > getMask() const
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
LLVM_NODISCARD bool empty() const
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
LLVM_NODISCARD bool empty() const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
LLVM_NODISCARD T pop_back_val()
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
iterator insert(iterator I, T &&Elt)
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
bool has(LibFunc F) const
Tests whether a library function is available.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
virtual bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
virtual bool hasAndNot(SDValue X) const
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
virtual bool enableAggressiveFMAFusion(EVT VT) const
Return true if target always benefits from combining into FMA for a given value type.
bool isIndexedStoreLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
SDValue promoteTargetBoolean(SelectionDAG &DAG, SDValue Bool, EVT ValVT) const
Promote the given target boolean to a target boolean of the given type.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual bool canCombineTruncStore(EVT ValVT, EVT MemVT, bool LegalOnly) const
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, bool *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
virtual bool convertSetCCLogicToBitwiseLogic(EVT VT) const
Use bitwise logic to make pairs of compares more efficient.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isVectorLoadExtDesirable(SDValue ExtVal) const
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
virtual bool hasBitPreservingFPLogic(EVT VT) const
Return true if it is safe to transform an integer-domain bitwise operation into the equivalent floati...
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual bool isFPExtFoldable(const MachineInstr &MI, unsigned Opcode, LLT DestTy, LLT SrcTy) const
Return true if an fpext operation input to an Opcode operation is free (for instance,...
virtual bool hasBitTest(SDValue X, SDValue Y) const
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
bool isTruncStoreLegal(EVT ValVT, EVT MemVT) const
Return true if the specified store with truncation is legal on this target.
virtual bool shouldRemoveExtendFromGSIndex(EVT IndexVT, EVT DataVT) const
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
virtual bool generateFMAsInMachineCombiner(EVT VT, CodeGenOpt::Level OptLevel) const
virtual bool isFPImmLegal(const APFloat &, EVT, bool ForCodeSize=false) const
Returns true if the target can instruction select the specified FP immediate natively.
virtual bool isExtractVecEltCheap(EVT VT, unsigned Index) const
Return true if extraction of a scalar element from the given vector type at the given index is cheap.
virtual bool shouldNormalizeToSelectSequence(LLVMContext &Context, EVT VT) const
Returns true if we should normalize select(N0&N1, X, Y) => select(N0, select(N1, X,...
bool isIndexedMaskedLoadLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
virtual bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
virtual bool isFsqrtCheap(SDValue X, SelectionDAG &DAG) const
Return true if SQRT(X) shouldn't be replaced with X*RSQRT(X).
virtual bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to fold a pair of shifts into a mask.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool shouldAvoidTransformToShift(EVT VT, unsigned Amount) const
Return true if creating a shift of the type by the given amount is not profitable.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
virtual bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal on this target.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual bool preferIncOfAddToSubOfNot(EVT VT) const
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
virtual bool isLegalAddImmediate(int64_t) const
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const
Return true if it is profitable to reduce a load to a smaller type.
virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const
virtual bool isFNegFree(EVT VT) const
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool mergeStoresAfterLegalization(EVT MemVT) const
Allow store merging for the specified type after legalization in addition to before legalization.
virtual bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const
Return true if it is cheaper to split the store of a merged int val from a pair of smaller values int...
bool isLoadExtLegalOrCustom(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal or custom on this target.
virtual bool storeOfVectorConstantIsCheap(EVT MemVT, unsigned NumElem, unsigned AddrSpace) const
Return true if it is expected to be cheaper to do a store of a non-zero vector constant with the give...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
virtual bool shouldFoldMaskToVariableShiftPair(SDValue X) const
There are two ways to clear extreme bits (either low or high): Mask: x & (-1 << y) (the instcombine c...
bool isIndexedLoadLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
virtual bool canMergeStoresTo(unsigned AS, EVT MemVT, const MachineFunction &MF) const
Returns if it's reasonable to merge stores to MemVT size.
bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal on this target.
virtual bool shouldScalarizeBinop(SDValue VecOp) const
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: (store (y (conv x)), y*)) -> (store x,...
bool isIndexedMaskedStoreLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
virtual bool isVectorClearMaskLegal(ArrayRef< int >, EVT) const
Similar to isShuffleMaskLegal.
bool hasTargetDAGCombine(ISD::NodeType NT) const
If true, the target has custom DAG combine transformations that it can perform for the specified node...
virtual bool shouldSplatInsEltVarIndex(EVT) const
Return true if inserting a scalar into a variable element of an undef vector is more efficiently hand...
NegatibleCost
Enum that specifies when a float negation is beneficial.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual bool isFMADLegal(const MachineInstr &MI, LLT Ty) const
Returns true if MI can be combined with another instruction to form TargetOpcode::G_FMAD.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
virtual bool isNarrowingProfitable(EVT, EVT) const
Return true if it's profitable to narrow operations of type VT1 to VT2.
virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const
virtual bool isFAbsFree(EVT VT) const
Return true if an fabs operation is free to the point where it is never worthwhile to replace it with...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
virtual bool hasPairedLoad(EVT, Align &) const
Return true if the target supplies and combines to a paired load two loaded values of type LoadedType...
virtual bool convertSelectOfConstantsToMath(EVT VT) const
Return true if a select of constants (select Cond, C1, C2) should be transformed into simple math ops...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
virtual bool isReassocProfitable(SelectionDAG &DAG, SDValue N0, SDValue N1) const
SDValue getCheaperNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, unsigned Depth=0) const
This is the helper function to return the newly negated expression only when the cost is cheaper.
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "lookthrough" ops that don't contrib...
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
virtual bool IsDesirableToPromoteOp(SDValue, EVT &) const
This method query the target whether it is beneficial for dag combiner to promote the specified node.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth=0) const
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool isConstFalseVal(SDValue N) const
Return if the N is a constant or constant vector equal to the false value from getBooleanContents().
SDValue BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::UDIV node expressing a divide by constant, return a DAG expression to select that will ...
virtual bool getPostIndexedAddressParts(SDNode *, SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
virtual bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to move this shift by a constant amount through its operand,...
virtual unsigned combineRepeatedFPDivisors() const
Indicate whether this target prefers to combine FDIVs with the same divisor.
virtual bool getPreIndexedAddressParts(SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::SDIV node expressing a divide by constant, return a DAG expression to select that will ...
virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SDIV lowering for power-of-2 denominators.
virtual SDValue BuildSREMPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SREM lowering for power-of-2 denominators.
virtual bool isDesirableToTransformToIntegerOp(unsigned, EVT) const
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
unsigned NoSignedZerosFPMath
NoSignedZerosFPMath - This flag is enabled when the -enable-no-signed-zeros-fp-math is specified on t...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual bool useAA() const
Enable use of alias analysis during code generation (during MI scheduling, DAGCombine,...
ScalarTy getFixedSize() const
static TypeSize Fixed(ScalarTy MinVal)
The instances of the Type class are immutable: once they are created, they are never changed.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
This class is used to represent EVT's, which are used to parameterize some operations.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
iterator_range< use_iterator > uses()
int getNumOccurrences() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
const APInt & umin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be unsigned.
const APInt & umax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be unsigned.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Fast - This calling convention attempts to make calls as fast as possible (e.g.
@ C
C - The default llvm calling convention, compatible with C.
CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical AND between different comparisons of identical values: ((X op1 Y) & (X...
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ ADDCARRY
Carry-using nodes for multiple precision addition and subtraction.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ SSUBO
Same for subtraction.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ CARRY_FALSE
CARRY_FALSE - This node is used when folding other nodes, like ADDC/SUBC, which indicate the carry re...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isIndexTypeSigned(MemIndexType IndexType)
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool matchUnaryPredicate(SDValue Op, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant BUI...
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
bool isExtOpcode(unsigned Opcode)
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
Optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
bool isVPBinaryOp(unsigned Opcode)
Whether this is a vector-predicated binary operation opcode.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
Optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
bool matchBinaryPredicate(SDValue LHS, SDValue RHS, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
bool isVPReduction(unsigned Opcode)
Whether this is a vector-predicated reduction opcode.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical OR between different comparisons of identical values: ((X op1 Y) | (X ...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
@ Undef
Value of the register doesn't matter.
initializer< Ty > init(const Ty &Val)
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
bool operator<(int64_t V1, const APSInt &V2)
void stable_sort(R &&Range)
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool operator==(uint64_t V1, const APInt &V2)
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs)
If V is a bitwise not, returns the inverted operand.
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
llvm::SmallVector< int, 16 > createUnaryMask(ArrayRef< int > Mask, unsigned NumElts)
Given a shuffle mask for a binary shuffle, create the equivalent shuffle mask assuming both operands ...
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
bool getAlign(const Function &F, unsigned index, unsigned &align)
bool operator!=(uint64_t V1, const APInt &V2)
bool operator>=(int64_t V1, const APSInt &V2)
std::string & operator+=(std::string &buffer, StringRef string)
LLVM_NODISCARD decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_NODISCARD decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
bool widenShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Try to transform a shuffle mask by replacing elements with the scaled index for an equivalent mask of...
auto reverse(ContainerTy &&C, std::enable_if_t< has_rbegin< ContainerTy >::value > *=nullptr)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
LLVM_NODISCARD bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool isMinSignedConstant(SDValue V)
Returns true if V is a constant min signed integer value.
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
unsigned M1(unsigned Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool isConstantOrConstantVector(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowFP=true, bool AllowOpaqueConstants=true)
Return true if the specified instruction is known to be a constant, or a vector of constants.
bool operator>(int64_t V1, const APSInt &V2)
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
unsigned countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
unsigned countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1.
detail::ValueMatchesPoly< M > HasValue(M Matcher)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
void sort(IteratorTy Start, IteratorTy End)
SDValue peekThroughOneUseBitcasts(SDValue V)
Return the non-bitcasted and one-use source operand of V if it exists.
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
unsigned countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
@ Z
zlib style complession
bool is_splat(R &&Range)
Wrapper function around std::equal to detect if all elements in a container are same.
void narrowShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Replace each shuffle mask index with the scaled sequential indices for an equivalent mask of narrowed...
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
unsigned M0(unsigned Val)
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
unsigned Log2(Align A)
Returns the log2 of the alignment.
bool operator<=(int64_t V1, const APSInt &V2)
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
int getSplatIndex(ArrayRef< int > Mask)
If all non-negative Mask elements are the same value, return that value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
AAMDNodes concat(const AAMDNodes &Other) const
Determine the best AAMDNodes after concatenating two different locations together.
static constexpr roundingMode rmNearestTiesToEven
static unsigned int semanticsPrecision(const fltSemantics &)
opStatus
IEEE-754R 7: Default exception handling.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Represent subnormal handling kind for floating point instruction inputs and outputs.
static constexpr DenormalMode getIEEE()
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
EVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isRound() const
Return true if the size is a power-of-two number of bytes.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isZeroSized() const
Test if the given EVT has zero size, this will fail if called on a scalable type.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
Helper struct to store a base, index and offset that forms an address.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
bool isAllOnes() const
Returns true if value is all one bits.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasAllowReassociation() const
void setNoUnsignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...
void AddToWorklist(SDNode *N)
bool recursivelyDeleteUnusedNodes(SDNode *N)
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...